970 lines
33 KiB
C#
970 lines
33 KiB
C#
using System;
|
||
using System.Collections.Generic;
|
||
using System.Linq;
|
||
using System.Globalization;
|
||
using System.Text;
|
||
using System.Threading;
|
||
using ClosedXML.Excel;
|
||
using NPOI.HSSF.UserModel;
|
||
using NPOI.SS.UserModel;
|
||
using NPOI.SS.Util;
|
||
|
||
namespace ExcelMatcher;
|
||
|
||
public sealed class TableData
|
||
{
|
||
public string FilePath { get; init; } = "";
|
||
public string SheetName { get; init; } = "";
|
||
public List<string> Columns { get; init; } = new();
|
||
public List<string[]> Rows { get; init; } = new();
|
||
public int RowCount => Rows.Count;
|
||
}
|
||
|
||
public static class ExcelReader
|
||
{
|
||
public static List<TableData> LoadWorkbook(string path)
|
||
{
|
||
var ext = Path.GetExtension(path).ToLowerInvariant();
|
||
return ext is ".xls" or ".xlt" ? LoadXls(path) : LoadXlsx(path);
|
||
}
|
||
|
||
private static List<TableData> LoadXlsx(string path)
|
||
{
|
||
var result = new List<TableData>();
|
||
using var wb = new XLWorkbook(path);
|
||
foreach (var ws in wb.Worksheets)
|
||
{
|
||
result.Add(LoadSheet(path, ws));
|
||
}
|
||
return result;
|
||
}
|
||
|
||
private static List<TableData> LoadXls(string path)
|
||
{
|
||
var result = new List<TableData>();
|
||
using var fs = File.OpenRead(path);
|
||
var wb = new HSSFWorkbook(fs);
|
||
var fmt = new DataFormatter();
|
||
for (int i = 0; i < wb.NumberOfSheets; i++)
|
||
{
|
||
var ws = wb.GetSheetAt(i);
|
||
if (ws is null) continue;
|
||
result.Add(LoadXlsSheet(path, ws, fmt));
|
||
}
|
||
return result;
|
||
}
|
||
|
||
private static string CellText(ISheet ws, int r, int c, DataFormatter fmt)
|
||
{
|
||
var cell = ws.GetRow(r)?.GetCell(c);
|
||
return cell is null ? "" : (fmt.FormatCellValue(cell) ?? "");
|
||
}
|
||
|
||
private static TableData LoadXlsSheet(string path, ISheet ws, DataFormatter fmt)
|
||
{
|
||
int firstRow = -1, lastRow = -1, firstCol = -1, lastCol = -1;
|
||
|
||
for (int r = 0; r <= ws.LastRowNum; r++)
|
||
{
|
||
var row = ws.GetRow(r);
|
||
if (row is null) continue;
|
||
int minCol = -1, maxCol = -1;
|
||
foreach (var cell in row.Cells)
|
||
{
|
||
if (string.IsNullOrWhiteSpace(fmt.FormatCellValue(cell))) continue;
|
||
int c = cell.ColumnIndex;
|
||
if (minCol == -1 || c < minCol) minCol = c;
|
||
if (maxCol == -1 || c > maxCol) maxCol = c;
|
||
}
|
||
if (minCol == -1) continue;
|
||
if (firstRow == -1) firstRow = r;
|
||
lastRow = r;
|
||
if (firstCol == -1 || minCol < firstCol) firstCol = minCol;
|
||
if (lastCol == -1 || maxCol > lastCol) lastCol = maxCol;
|
||
}
|
||
|
||
if (firstRow == -1)
|
||
return new TableData { FilePath = path, SheetName = ws.SheetName };
|
||
|
||
int colCount = lastCol - firstCol + 1;
|
||
var cols = new List<string>();
|
||
for (int c = 0; c < colCount; c++)
|
||
{
|
||
var header = CellText(ws, firstRow, firstCol + c, fmt);
|
||
cols.Add(string.IsNullOrEmpty(header) ? $"Колонка {c + 1}" : header);
|
||
}
|
||
|
||
var rows = new List<string[]>();
|
||
for (int r = firstRow + 1; r <= lastRow; r++)
|
||
{
|
||
var row = new string[colCount];
|
||
bool any = false;
|
||
for (int c = 0; c < colCount; c++)
|
||
{
|
||
var cell = CellText(ws, r, firstCol + c, fmt);
|
||
row[c] = cell;
|
||
if (!string.IsNullOrWhiteSpace(cell)) any = true;
|
||
}
|
||
if (any) rows.Add(row);
|
||
}
|
||
|
||
return new TableData
|
||
{
|
||
FilePath = path,
|
||
SheetName = ws.SheetName,
|
||
Columns = cols,
|
||
Rows = rows
|
||
};
|
||
}
|
||
|
||
private static TableData LoadSheet(string path, IXLWorksheet ws)
|
||
{
|
||
var cols = new List<string>();
|
||
var rows = new List<string[]>();
|
||
|
||
var firstUsed = ws.FirstCellUsed();
|
||
var lastUsed = ws.LastCellUsed();
|
||
if (firstUsed == null || lastUsed == null)
|
||
return new TableData { FilePath = path, SheetName = ws.Name };
|
||
|
||
int firstRow = firstUsed.Address.RowNumber;
|
||
int lastRow = lastUsed.Address.RowNumber;
|
||
int firstCol = firstUsed.Address.ColumnNumber;
|
||
int lastCol = lastUsed.Address.ColumnNumber;
|
||
int colCount = lastCol - firstCol + 1;
|
||
|
||
for (int c = 0; c < colCount; c++)
|
||
{
|
||
var header = ws.Cell(firstRow, firstCol + c).GetString().Trim();
|
||
cols.Add(string.IsNullOrEmpty(header) ? $"Колонка {c + 1}" : header);
|
||
}
|
||
|
||
for (int r = firstRow + 1; r <= lastRow; r++)
|
||
{
|
||
var row = new string[colCount];
|
||
bool any = false;
|
||
for (int c = 0; c < colCount; c++)
|
||
{
|
||
var cell = ws.Cell(r, firstCol + c).GetString();
|
||
row[c] = cell;
|
||
if (!string.IsNullOrWhiteSpace(cell)) any = true;
|
||
}
|
||
if (any) rows.Add(row);
|
||
}
|
||
|
||
return new TableData
|
||
{
|
||
FilePath = path,
|
||
SheetName = ws.Name,
|
||
Columns = cols,
|
||
Rows = rows
|
||
};
|
||
}
|
||
}
|
||
|
||
public enum MatchMode
|
||
{
|
||
KeepMatches,
|
||
KeepOnlyMain,
|
||
KeepOnlySecondary,
|
||
KeepBoth
|
||
}
|
||
|
||
public enum MatchKind
|
||
{
|
||
Exact,
|
||
Partial,
|
||
Fuzzy
|
||
}
|
||
|
||
public sealed class MatchOptions
|
||
{
|
||
public bool[] MainUsed = Array.Empty<bool>();
|
||
public bool[] SecondaryUsed = Array.Empty<bool>();
|
||
public bool IgnoreCaseAndSpaces = true;
|
||
public bool ExtendedNormalize = false;
|
||
public MatchKind Kind = MatchKind.Exact;
|
||
public int FuzzyThreshold = 85;
|
||
public bool UseOr = false;
|
||
public IReadOnlyDictionary<string, string>? Synonyms;
|
||
public CancellationToken Token = default;
|
||
|
||
public MatchOptions Clone() => (MatchOptions)MemberwiseClone();
|
||
}
|
||
|
||
public sealed class MatchPair
|
||
{
|
||
public string[] RowA { get; init; } = Array.Empty<string>();
|
||
public string[] RowB { get; init; } = Array.Empty<string>();
|
||
public string Type { get; init; } = "";
|
||
public double Score { get; init; }
|
||
public int BCount { get; init; }
|
||
}
|
||
|
||
public sealed class MatchResult
|
||
{
|
||
public List<string[]> Matches { get; init; } = new();
|
||
public List<string[]> OnlyMain { get; init; } = new();
|
||
public List<string[]> OnlySecondary { get; init; } = new();
|
||
public List<MatchPair> Pairs { get; init; } = new();
|
||
}
|
||
|
||
public sealed class MergeOutput
|
||
{
|
||
public List<string> Columns { get; init; } = new();
|
||
public List<string[]> Rows { get; init; } = new();
|
||
}
|
||
|
||
public static class TableMatcher
|
||
{
|
||
private static List<int> KeyIndexes(bool[] used)
|
||
{
|
||
return used.Select((u, i) => (u, i)).Where(x => x.u).Select(x => x.i).ToList();
|
||
}
|
||
|
||
public static double Similarity(string a, string b)
|
||
{
|
||
if (a == b) return 1;
|
||
if (a.Length == 0) return 0;
|
||
if (b.Length == 0) return 0;
|
||
int dist = Levenshtein(a, b);
|
||
int maxLen = Math.Max(a.Length, b.Length);
|
||
return 1.0 - (double)dist / maxLen;
|
||
}
|
||
|
||
private static int Levenshtein(string a, string b)
|
||
{
|
||
var prev = new int[b.Length + 1];
|
||
var curr = new int[b.Length + 1];
|
||
for (int j = 0; j <= b.Length; j++) prev[j] = j;
|
||
for (int i = 1; i <= a.Length; i++)
|
||
{
|
||
curr[0] = i;
|
||
for (int j = 1; j <= b.Length; j++)
|
||
{
|
||
int cost = a[i - 1] == b[j - 1] ? 0 : 1;
|
||
curr[j] = Math.Min(Math.Min(curr[j - 1] + 1, prev[j] + 1), prev[j - 1] + cost);
|
||
}
|
||
(prev, curr) = (curr, prev);
|
||
}
|
||
return prev[b.Length];
|
||
}
|
||
|
||
private static string NormalizeBase(string s, IReadOnlyDictionary<string, string>? synonyms)
|
||
{
|
||
var t = s;
|
||
if (synonyms is not null)
|
||
{
|
||
foreach (var kv in synonyms)
|
||
t = t.Replace(kv.Key, kv.Value, StringComparison.OrdinalIgnoreCase);
|
||
}
|
||
return t.Replace(" ", "").Replace("\t", "").ToLowerInvariant().Replace('ё', 'e');
|
||
}
|
||
|
||
private static string NormalizeCore(string s)
|
||
{
|
||
var sb = new StringBuilder(s.Length);
|
||
foreach (char ch in s.ToLowerInvariant())
|
||
{
|
||
if (char.IsLetterOrDigit(ch)) sb.Append(ch);
|
||
}
|
||
var clean = sb.ToString().Replace('ё', 'е');
|
||
foreach (var p in new[] { "ооо", "зао", "оао", "тоо", "пао" })
|
||
{
|
||
if (clean.StartsWith(p, StringComparison.Ordinal) && clean.Length > p.Length + 1)
|
||
{
|
||
clean = clean.Substring(p.Length);
|
||
break;
|
||
}
|
||
}
|
||
return clean;
|
||
}
|
||
|
||
private static string NormValue(string s, bool ignore, MatchOptions o)
|
||
{
|
||
if (!ignore) return s;
|
||
return o.ExtendedNormalize ? NormalizeCore(s) : NormalizeBase(s, o.Synonyms);
|
||
}
|
||
|
||
private static string[] RowParts(string[] row, List<int> keyIndexes, int take, MatchOptions o)
|
||
{
|
||
var parts = new string[take];
|
||
for (int i = 0; i < take; i++)
|
||
parts[i] = keyIndexes[i] < row.Length ? NormValue(row[keyIndexes[i]], o.IgnoreCaseAndSpaces, o) : "";
|
||
return parts;
|
||
}
|
||
|
||
private static bool IsEmptyParts(string[] parts) => parts.All(string.IsNullOrEmpty);
|
||
|
||
private static string JoinKey(string[] parts) => string.Join("\u00a6", parts);
|
||
|
||
private static bool PartsContain(string[] a, string[] b)
|
||
{
|
||
for (int i = 0; i < a.Length; i++)
|
||
{
|
||
var x = a[i];
|
||
var y = b[i];
|
||
if (string.IsNullOrEmpty(x) || string.IsNullOrEmpty(y)) return false;
|
||
if (x.Contains(y, StringComparison.Ordinal) || y.Contains(x, StringComparison.Ordinal)) continue;
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
private static bool AnyColContains(string[] a, string[] b)
|
||
{
|
||
for (int i = 0; i < a.Length; i++)
|
||
{
|
||
var x = a[i];
|
||
var y = b[i];
|
||
if (string.IsNullOrEmpty(x) || string.IsNullOrEmpty(y)) continue;
|
||
if (x.Contains(y, StringComparison.Ordinal) || y.Contains(x, StringComparison.Ordinal)) return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
private static double ColumnSim(string[] a, string[] b, bool orMode)
|
||
{
|
||
double sum = 0;
|
||
int n = 0;
|
||
double best = 0;
|
||
for (int i = 0; i < a.Length; i++)
|
||
{
|
||
if (a[i].Length == 0 && b[i].Length == 0 && !orMode)
|
||
{
|
||
sum += 1;
|
||
n++;
|
||
continue;
|
||
}
|
||
if (a[i].Length == 0 || b[i].Length == 0)
|
||
{
|
||
if (orMode) continue;
|
||
sum += 0;
|
||
n++;
|
||
continue;
|
||
}
|
||
var s = Similarity(a[i], b[i]);
|
||
if (orMode)
|
||
{
|
||
if (s > best) best = s;
|
||
}
|
||
else
|
||
{
|
||
sum += s;
|
||
n++;
|
||
}
|
||
}
|
||
if (orMode) return best;
|
||
return n == 0 ? 0 : sum / n;
|
||
}
|
||
|
||
private sealed class Candidate
|
||
{
|
||
public string[]? BRow;
|
||
public string Type = "";
|
||
public double Score;
|
||
public int BCount;
|
||
}
|
||
|
||
private static List<Candidate> MatchMain(TableData main, TableData secondary, MatchOptions o, out List<string[]> onlySecondary)
|
||
{
|
||
var mainIdx = KeyIndexes(o.MainUsed);
|
||
var secIdx = KeyIndexes(o.SecondaryUsed);
|
||
int keyCount = Math.Min(mainIdx.Count, secIdx.Count);
|
||
onlySecondary = new List<string[]>();
|
||
|
||
if (keyCount == 0)
|
||
return main.Rows.Select(_ => (Candidate?)null).Cast<Candidate>().ToList();
|
||
|
||
var token = o.Token;
|
||
bool or = o.UseOr;
|
||
var candidates = new List<Candidate>(main.RowCount);
|
||
|
||
if (o.Kind == MatchKind.Exact)
|
||
{
|
||
if (!or)
|
||
{
|
||
var mainKeys = new HashSet<string>(StringComparer.Ordinal);
|
||
foreach (var row in main.Rows)
|
||
{
|
||
var parts = RowParts(row, mainIdx, keyCount, o);
|
||
if (!IsEmptyParts(parts)) mainKeys.Add(JoinKey(parts));
|
||
}
|
||
|
||
var bByKey = new Dictionary<string, (string[] Row, int Count)>(StringComparer.Ordinal);
|
||
foreach (var row in secondary.Rows)
|
||
{
|
||
token.ThrowIfCancellationRequested();
|
||
var parts = RowParts(row, secIdx, keyCount, o);
|
||
if (IsEmptyParts(parts)) continue;
|
||
var k = JoinKey(parts);
|
||
if (bByKey.TryGetValue(k, out var cur))
|
||
bByKey[k] = (cur.Row, cur.Count + 1);
|
||
else
|
||
bByKey[k] = (row, 1);
|
||
}
|
||
|
||
foreach (var row in main.Rows)
|
||
{
|
||
token.ThrowIfCancellationRequested();
|
||
var parts = RowParts(row, mainIdx, keyCount, o);
|
||
if (IsEmptyParts(parts))
|
||
{
|
||
candidates.Add(new Candidate());
|
||
continue;
|
||
}
|
||
if (bByKey.TryGetValue(JoinKey(parts), out var hit))
|
||
{
|
||
candidates.Add(new Candidate { BRow = hit.Row, Type = "точное", Score = 100, BCount = hit.Count });
|
||
}
|
||
else
|
||
{
|
||
candidates.Add(new Candidate());
|
||
}
|
||
}
|
||
|
||
foreach (var row in secondary.Rows)
|
||
{
|
||
token.ThrowIfCancellationRequested();
|
||
var parts = RowParts(row, secIdx, keyCount, o);
|
||
if (!IsEmptyParts(parts) && !mainKeys.Contains(JoinKey(parts)))
|
||
onlySecondary.Add(row);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
var mainSets = new HashSet<string>[keyCount];
|
||
for (int j = 0; j < keyCount; j++) mainSets[j] = new HashSet<string>(StringComparer.Ordinal);
|
||
foreach (var row in main.Rows)
|
||
{
|
||
var parts = RowParts(row, mainIdx, keyCount, o);
|
||
if (IsEmptyParts(parts)) continue;
|
||
for (int j = 0; j < keyCount; j++)
|
||
if (parts[j].Length > 0) mainSets[j].Add(parts[j]);
|
||
}
|
||
|
||
var bByVal = new Dictionary<string, (string[] Row, int Count)>[keyCount];
|
||
for (int j = 0; j < keyCount; j++) bByVal[j] = new Dictionary<string, (string[] Row, int Count)>(StringComparer.Ordinal);
|
||
foreach (var row in secondary.Rows)
|
||
{
|
||
token.ThrowIfCancellationRequested();
|
||
var parts = RowParts(row, secIdx, keyCount, o);
|
||
if (IsEmptyParts(parts)) continue;
|
||
for (int j = 0; j < keyCount; j++)
|
||
{
|
||
if (parts[j].Length == 0) continue;
|
||
var v = parts[j];
|
||
if (bByVal[j].TryGetValue(v, out var cur))
|
||
bByVal[j][v] = (cur.Row, cur.Count + 1);
|
||
else
|
||
bByVal[j][v] = (row, 1);
|
||
}
|
||
}
|
||
|
||
foreach (var row in main.Rows)
|
||
{
|
||
token.ThrowIfCancellationRequested();
|
||
var parts = RowParts(row, mainIdx, keyCount, o);
|
||
if (IsEmptyParts(parts))
|
||
{
|
||
candidates.Add(new Candidate());
|
||
continue;
|
||
}
|
||
string[]? bestRow = null;
|
||
int bestCount = 0;
|
||
for (int j = 0; j < keyCount; j++)
|
||
{
|
||
if (parts[j].Length == 0) continue;
|
||
if (bByVal[j].TryGetValue(parts[j], out var hit) && hit.Count > bestCount)
|
||
{
|
||
bestRow = hit.Row;
|
||
bestCount = hit.Count;
|
||
}
|
||
}
|
||
if (bestRow is not null)
|
||
candidates.Add(new Candidate { BRow = bestRow, Type = "точное", Score = 100, BCount = bestCount });
|
||
else
|
||
candidates.Add(new Candidate());
|
||
}
|
||
|
||
foreach (var row in secondary.Rows)
|
||
{
|
||
token.ThrowIfCancellationRequested();
|
||
var parts = RowParts(row, secIdx, keyCount, o);
|
||
if (IsEmptyParts(parts)) continue;
|
||
bool matched = false;
|
||
for (int j = 0; j < keyCount; j++)
|
||
{
|
||
if (parts[j].Length > 0 && mainSets[j].Contains(parts[j]))
|
||
{
|
||
matched = true;
|
||
break;
|
||
}
|
||
}
|
||
if (!matched) onlySecondary.Add(row);
|
||
}
|
||
}
|
||
return candidates;
|
||
}
|
||
|
||
// Частичное / Нечёткое: строим списки частей и сканируем строку за строкой.
|
||
var secRows = secondary.Rows
|
||
.Select(row => (Row: row, Parts: RowParts(row, secIdx, keyCount, o)))
|
||
.ToList();
|
||
|
||
var hits = new bool[secRows.Count];
|
||
|
||
foreach (var row in main.Rows)
|
||
{
|
||
token.ThrowIfCancellationRequested();
|
||
var parts = RowParts(row, mainIdx, keyCount, o);
|
||
if (IsEmptyParts(parts))
|
||
{
|
||
candidates.Add(new Candidate());
|
||
continue;
|
||
}
|
||
|
||
string[]? bestRow = null;
|
||
double bestScore = 0;
|
||
int hitCount = 0;
|
||
|
||
if (o.Kind == MatchKind.Partial)
|
||
{
|
||
for (int i = 0; i < secRows.Count; i++)
|
||
{
|
||
var b = secRows[i];
|
||
bool ok = or ? AnyColContains(parts, b.Parts) : PartsContain(parts, b.Parts);
|
||
if (!ok) continue;
|
||
hits[i] = true;
|
||
hitCount++;
|
||
if (bestRow is null) bestRow = b.Row;
|
||
}
|
||
if (bestRow is not null)
|
||
candidates.Add(new Candidate { BRow = bestRow, BCount = hitCount, Score = 100, Type = "частичное" });
|
||
else
|
||
candidates.Add(new Candidate());
|
||
}
|
||
else
|
||
{
|
||
double threshold = o.FuzzyThreshold / 100.0;
|
||
for (int i = 0; i < secRows.Count; i++)
|
||
{
|
||
var b = secRows[i];
|
||
double sim = ColumnSim(parts, b.Parts, or);
|
||
if (sim >= threshold)
|
||
{
|
||
hits[i] = true;
|
||
hitCount++;
|
||
if (sim > bestScore)
|
||
{
|
||
bestScore = sim;
|
||
bestRow = b.Row;
|
||
}
|
||
}
|
||
}
|
||
if (bestRow is not null)
|
||
candidates.Add(new Candidate
|
||
{
|
||
BRow = bestRow,
|
||
BCount = hitCount,
|
||
Score = Math.Round(bestScore * 100, 1),
|
||
Type = "нечёткое"
|
||
});
|
||
else
|
||
candidates.Add(new Candidate());
|
||
}
|
||
}
|
||
|
||
for (int i = 0; i < secRows.Count; i++)
|
||
{
|
||
token.ThrowIfCancellationRequested();
|
||
if (!hits[i]) onlySecondary.Add(secRows[i].Row);
|
||
}
|
||
|
||
return candidates;
|
||
}
|
||
|
||
public static MatchResult RunAll(TableData main, TableData secondary, MatchOptions o)
|
||
{
|
||
var candidates = MatchMain(main, secondary, o, out var onlySecondary);
|
||
var matches = new List<string[]>();
|
||
var pairs = new List<MatchPair>();
|
||
var onlyMain = new List<string[]>();
|
||
|
||
int n = Math.Min(candidates.Count, main.RowCount);
|
||
for (int i = 0; i < n; i++)
|
||
{
|
||
o.Token.ThrowIfCancellationRequested();
|
||
var c = candidates[i];
|
||
if (c.BRow is null)
|
||
{
|
||
onlyMain.Add(main.Rows[i]);
|
||
continue;
|
||
}
|
||
matches.Add(main.Rows[i]);
|
||
pairs.Add(new MatchPair
|
||
{
|
||
RowA = main.Rows[i],
|
||
RowB = c.BRow,
|
||
Type = c.Type,
|
||
Score = c.Score,
|
||
BCount = c.BCount
|
||
});
|
||
}
|
||
|
||
return new MatchResult
|
||
{
|
||
Matches = matches,
|
||
Pairs = pairs,
|
||
OnlyMain = onlyMain,
|
||
OnlySecondary = onlySecondary
|
||
};
|
||
}
|
||
|
||
public static MatchResult RunAll(TableData main, TableData secondary, bool[] mainUsed, bool[] secondaryUsed, bool ignore)
|
||
=> RunAll(main, secondary, new MatchOptions { MainUsed = mainUsed, SecondaryUsed = secondaryUsed, IgnoreCaseAndSpaces = ignore });
|
||
|
||
public static MatchResult RunAll(TableData main, TableData secondary, bool[] mainUsed, bool[] secondaryUsed, bool ignore, bool partial)
|
||
=> RunAll(main, secondary, new MatchOptions
|
||
{
|
||
MainUsed = mainUsed,
|
||
SecondaryUsed = secondaryUsed,
|
||
IgnoreCaseAndSpaces = ignore,
|
||
Kind = partial ? MatchKind.Partial : MatchKind.Exact
|
||
});
|
||
|
||
public static MergeOutput Merge(TableData main, TableData secondary, MatchOptions o, bool removeIdenticalColumns = true)
|
||
{
|
||
var candidates = MatchMain(main, secondary, o, out _);
|
||
|
||
var columns = new List<string>(main.Columns);
|
||
var extraCols = new List<int>();
|
||
|
||
if (removeIdenticalColumns)
|
||
{
|
||
for (int j = 0; j < secondary.Columns.Count; j++)
|
||
{
|
||
if (columns.Contains(secondary.Columns[j]))
|
||
continue;
|
||
|
||
var target = SortedColumnValues(secondary.Rows, j);
|
||
bool duplicate = false;
|
||
for (int c = 0; c < main.Columns.Count && !duplicate; c++)
|
||
duplicate = ColumnValuesEqual(target, SortedColumnValues(main.Rows, c));
|
||
for (int e = 0; e < extraCols.Count && !duplicate; e++)
|
||
duplicate = ColumnValuesEqual(target, SortedColumnValues(secondary.Rows, extraCols[e]));
|
||
if (duplicate) continue;
|
||
|
||
columns.Add(secondary.Columns[j]);
|
||
extraCols.Add(j);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
for (int j = 0; j < secondary.Columns.Count; j++)
|
||
{
|
||
columns.Add(secondary.Columns[j]);
|
||
extraCols.Add(j);
|
||
}
|
||
}
|
||
|
||
string[] BuildMerged(string[] row, string[] secRow)
|
||
{
|
||
var merged = new string[columns.Count];
|
||
for (int c = 0; c < main.Columns.Count; c++)
|
||
merged[c] = c < row.Length ? row[c] : "";
|
||
for (int e = 0; e < extraCols.Count; e++)
|
||
{
|
||
int j = extraCols[e];
|
||
merged[main.Columns.Count + e] = j < secRow.Length ? secRow[j] : "";
|
||
}
|
||
return merged;
|
||
}
|
||
|
||
var rows = new List<string[]>();
|
||
int n = Math.Min(candidates.Count, main.RowCount);
|
||
for (int i = 0; i < n; i++)
|
||
{
|
||
o.Token.ThrowIfCancellationRequested();
|
||
var c = candidates[i];
|
||
if (c.BRow is not null)
|
||
rows.Add(BuildMerged(main.Rows[i], c.BRow));
|
||
}
|
||
|
||
return new MergeOutput { Columns = columns, Rows = rows };
|
||
}
|
||
|
||
public static MergeOutput Merge(
|
||
TableData main,
|
||
TableData secondary,
|
||
bool[] mainUsed,
|
||
bool[] secondaryUsed,
|
||
bool ignoreCaseAndSpaces,
|
||
bool partial = false,
|
||
bool removeIdenticalColumns = true)
|
||
=> Merge(main, secondary, new MatchOptions
|
||
{
|
||
MainUsed = mainUsed,
|
||
SecondaryUsed = secondaryUsed,
|
||
IgnoreCaseAndSpaces = ignoreCaseAndSpaces,
|
||
Kind = partial ? MatchKind.Partial : MatchKind.Exact
|
||
}, removeIdenticalColumns);
|
||
|
||
private static List<string> SortedColumnValues(List<string[]> rows, int colIndex)
|
||
{
|
||
var vals = new List<string>();
|
||
foreach (var r in rows)
|
||
vals.Add(colIndex < r.Length ? (r[colIndex] ?? "").Trim() : "");
|
||
vals.Sort(StringComparer.Ordinal);
|
||
return vals;
|
||
}
|
||
|
||
private static bool ColumnValuesEqual(List<string> a, List<string> b)
|
||
{
|
||
if (a.Count != b.Count) return false;
|
||
for (int i = 0; i < a.Count; i++)
|
||
if (!string.Equals(a[i], b[i], StringComparison.Ordinal)) return false;
|
||
return true;
|
||
}
|
||
|
||
public static (TableData Table, int Removed) Dedupe(TableData table, bool[] used, bool ignore, MatchOptions? template = null)
|
||
{
|
||
var idx = KeyIndexes(used);
|
||
if (idx.Count == 0)
|
||
return (table, 0);
|
||
|
||
var o = template?.Clone() ?? new MatchOptions();
|
||
o.MainUsed = used;
|
||
o.SecondaryUsed = used;
|
||
o.IgnoreCaseAndSpaces = ignore;
|
||
|
||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||
var rows = new List<string[]>();
|
||
int removed = 0;
|
||
foreach (var row in table.Rows)
|
||
{
|
||
o.Token.ThrowIfCancellationRequested();
|
||
var parts = RowParts(row, idx, idx.Count, o);
|
||
if (!IsEmptyParts(parts) && !seen.Add(JoinKey(parts)))
|
||
{
|
||
removed++;
|
||
continue;
|
||
}
|
||
rows.Add(row);
|
||
}
|
||
|
||
return (new TableData
|
||
{
|
||
FilePath = table.FilePath,
|
||
SheetName = table.SheetName,
|
||
Columns = table.Columns,
|
||
Rows = rows
|
||
}, removed);
|
||
}
|
||
|
||
public static (int EmptyKeys, int Total) KeyCoverage(TableData table, bool[] used, MatchOptions? o = null)
|
||
{
|
||
var idx = KeyIndexes(used);
|
||
int empty = 0;
|
||
foreach (var row in table.Rows)
|
||
{
|
||
bool allEmpty;
|
||
if (idx.Count == 0)
|
||
{
|
||
allEmpty = row.All(string.IsNullOrEmpty);
|
||
}
|
||
else
|
||
{
|
||
allEmpty = true;
|
||
foreach (var j in idx)
|
||
{
|
||
if (j >= row.Length || string.IsNullOrWhiteSpace(row[j]))
|
||
continue;
|
||
allEmpty = false;
|
||
break;
|
||
}
|
||
}
|
||
if (allEmpty) empty++;
|
||
}
|
||
return (empty, table.RowCount);
|
||
}
|
||
}
|
||
|
||
public static class ExcelExporter
|
||
{
|
||
public static void ExportAll(string path, TableData main, TableData secondary, MatchResult result, MergeOutput? merged = null)
|
||
{
|
||
using var wb = new XLWorkbook();
|
||
|
||
var wsMerged = wb.Worksheets.Add("Итог (совмещено)");
|
||
var wsPairs = wb.Worksheets.Add("Пары A↔B");
|
||
var wsMatches = wb.Worksheets.Add("Совпадения");
|
||
var wsOnlyMain = wb.Worksheets.Add("Только в A");
|
||
var wsOnlySecondary = wb.Worksheets.Add("Только в B");
|
||
var wsAnalytics = wb.Worksheets.Add("Аналитика");
|
||
|
||
if (merged is null || merged.Rows.Count == 0)
|
||
merged = new MergeOutput { Columns = main.Columns, Rows = result.Matches };
|
||
|
||
WriteSheet(wsMerged, merged.Columns, merged.Rows);
|
||
|
||
WritePairs(wsPairs, main, secondary, result);
|
||
|
||
WriteMatches(wsMatches, main, result);
|
||
|
||
WriteSheet(wsOnlyMain, main.Columns, result.OnlyMain);
|
||
WriteSheet(wsOnlySecondary, secondary.Columns, result.OnlySecondary);
|
||
|
||
WriteAnalytics(wsAnalytics, main, secondary, result);
|
||
|
||
wb.SaveAs(path);
|
||
}
|
||
|
||
private static void WriteMatches(IXLWorksheet ws, TableData main, MatchResult result)
|
||
{
|
||
var headers = new List<string>(main.Columns);
|
||
bool withMeta = result.Pairs.Count > 0 || main.Columns.Count == 0;
|
||
if (withMeta)
|
||
{
|
||
headers.Add("Тип");
|
||
headers.Add("Сходство %");
|
||
headers.Add("B-совпадений");
|
||
}
|
||
|
||
for (int c = 0; c < headers.Count; c++)
|
||
{
|
||
ws.Cell(1, c + 1).Value = headers[c];
|
||
ws.Cell(1, c + 1).Style.Font.Bold = true;
|
||
ws.Cell(1, c + 1).Style.Fill.BackgroundColor = XLColor.FromHtml("#DDEEFF");
|
||
}
|
||
|
||
int rows = result.Matches.Count;
|
||
for (int i = 0; i < rows; i++)
|
||
{
|
||
var row = result.Matches[i];
|
||
int baseCols = main.Columns.Count;
|
||
for (int c = 0; c < baseCols; c++)
|
||
{
|
||
SetCell(ws, i + 2, c + 1, c < row.Length ? row[c] : "");
|
||
}
|
||
if (withMeta)
|
||
{
|
||
var type = result.Pairs.Count > i ? result.Pairs[i].Type : "";
|
||
var score = result.Pairs.Count > i ? result.Pairs[i].Score : 0;
|
||
var count = result.Pairs.Count > i ? result.Pairs[i].BCount : 0;
|
||
SetCell(ws, i + 2, baseCols + 1, type);
|
||
SetCell(ws, i + 2, baseCols + 2, type == "нечёткое" ? score.ToString("0.#", CultureInfo.InvariantCulture) : "");
|
||
SetCell(ws, i + 2, baseCols + 3, count == 0 ? "" : count.ToString(CultureInfo.InvariantCulture));
|
||
}
|
||
}
|
||
|
||
ws.Columns().AdjustToContents();
|
||
if (rows == 0) ws.Row(2).Cell(1).Value = "Нет данных";
|
||
}
|
||
|
||
private static void WritePairs(IXLWorksheet ws, TableData main, TableData secondary, MatchResult result)
|
||
{
|
||
var headers = new List<string>();
|
||
foreach (var h in main.Columns) headers.Add("A: " + h);
|
||
headers.Add("Тип");
|
||
headers.Add("Сходство %");
|
||
headers.Add("B-совпадений");
|
||
foreach (var h in secondary.Columns) headers.Add("B: " + h);
|
||
|
||
for (int c = 0; c < headers.Count; c++)
|
||
{
|
||
ws.Cell(1, c + 1).Value = headers[c];
|
||
ws.Cell(1, c + 1).Style.Font.Bold = true;
|
||
ws.Cell(1, c + 1).Style.Fill.BackgroundColor = XLColor.FromHtml("#DDEEFF");
|
||
}
|
||
|
||
int aCols = main.Columns.Count;
|
||
int bCols = secondary.Columns.Count;
|
||
for (int i = 0; i < result.Pairs.Count; i++)
|
||
{
|
||
var p = result.Pairs[i];
|
||
var rowA = p.RowA;
|
||
var rowB = p.RowB;
|
||
for (int c = 0; c < aCols; c++)
|
||
SetCell(ws, i + 2, c + 1, c < rowA.Length ? rowA[c] : "");
|
||
SetCell(ws, i + 2, aCols + 1, p.Type);
|
||
SetCell(ws, i + 2, aCols + 2, p.Type == "нечёткое" ? p.Score.ToString("0.#", CultureInfo.InvariantCulture) : "");
|
||
SetCell(ws, i + 2, aCols + 3, p.BCount == 0 ? "" : p.BCount.ToString(CultureInfo.InvariantCulture));
|
||
for (int c = 0; c < bCols; c++)
|
||
SetCell(ws, i + 2, aCols + 3 + c + 1, c < rowB.Length ? rowB[c] : "");
|
||
}
|
||
|
||
ws.Columns().AdjustToContents();
|
||
if (result.Pairs.Count == 0) ws.Row(2).Cell(1).Value = "Нет данных";
|
||
}
|
||
|
||
private static void WriteAnalytics(IXLWorksheet ws, TableData main, TableData secondary, MatchResult result)
|
||
{
|
||
ws.Cell(1, 1).Value = "Показатель";
|
||
ws.Cell(1, 2).Value = "Значение";
|
||
ws.Cell(1, 1).Style.Font.Bold = true;
|
||
ws.Cell(1, 2).Style.Font.Bold = true;
|
||
|
||
int r = 2;
|
||
void Row(string name, object value)
|
||
{
|
||
ws.Cell(r, 1).Value = name;
|
||
ws.Cell(r, 2).Value = Convert.ToString(value, CultureInfo.InvariantCulture) ?? "";
|
||
r++;
|
||
}
|
||
|
||
Row("Всего в A", main.RowCount);
|
||
Row("Всего в B", secondary.RowCount);
|
||
Row("Совпадений (строк A)", result.Matches.Count);
|
||
Row("Только в A", result.OnlyMain.Count);
|
||
Row("Только в B", result.OnlySecondary.Count);
|
||
Row("Пар A↔B", result.Pairs.Count);
|
||
|
||
var exact = result.Pairs.Count(p => p.Type == "точное");
|
||
var partial = result.Pairs.Count(p => p.Type == "частичное");
|
||
var fuzzy = result.Pairs.Count(p => p.Type == "нечёткое");
|
||
Row("Из них точных", exact);
|
||
Row("Из них частичных", partial);
|
||
Row("Из них нечётких", fuzzy);
|
||
|
||
double avg = result.Pairs.Count == 0 ? 0 : Math.Round(result.Pairs.Average(p => p.BCount), 2);
|
||
int maxBC = result.Pairs.Count == 0 ? 0 : result.Pairs.Max(p => p.BCount);
|
||
Row("Среднее B-совпадений (N→M)", avg);
|
||
Row("Максимум B-совпадений (N→M)", maxBC);
|
||
|
||
ws.Columns().AdjustToContents();
|
||
}
|
||
|
||
private static void WriteSheet(IXLWorksheet ws, List<string> columns, List<string[]> rows)
|
||
{
|
||
int colCount = columns.Count;
|
||
for (int c = 0; c < colCount; c++)
|
||
{
|
||
ws.Cell(1, c + 1).Value = columns[c];
|
||
ws.Cell(1, c + 1).Style.Font.Bold = true;
|
||
ws.Cell(1, c + 1).Style.Fill.BackgroundColor = XLColor.FromHtml("#DDEEFF");
|
||
}
|
||
|
||
for (int i = 0; i < rows.Count; i++)
|
||
{
|
||
var row = rows[i];
|
||
for (int c = 0; c < row.Length && c < colCount; c++)
|
||
{
|
||
SetCell(ws, i + 2, c + 1, row[c]);
|
||
}
|
||
}
|
||
|
||
ws.Columns().AdjustToContents();
|
||
if (rows.Count == 0) ws.Row(2).Cell(1).Value = "Нет данных";
|
||
}
|
||
|
||
private static void SetCell(IXLWorksheet ws, int row, int col, string value)
|
||
{
|
||
var cell = ws.Cell(row, col);
|
||
if (double.TryParse(value, NumberStyles.Float | NumberStyles.AllowExponent, CultureInfo.InvariantCulture, out var d) &&
|
||
d.ToString(CultureInfo.InvariantCulture) == value)
|
||
cell.Value = d;
|
||
else
|
||
cell.Value = value;
|
||
}
|
||
} |