diff --git a/ExcelMatcher.csproj b/ExcelMatcher.csproj index f16588e..18d47ca 100644 --- a/ExcelMatcher.csproj +++ b/ExcelMatcher.csproj @@ -4,6 +4,7 @@ WinExe Exe net8.0 + 1.0.1 enable enable true diff --git a/ExcelService.cs b/ExcelService.cs index 6fb4152..2ddfbdf 100644 --- a/ExcelService.cs +++ b/ExcelService.cs @@ -213,6 +213,7 @@ public sealed class MergeOutput { public List Columns { get; init; } = new(); public List Rows { get; init; } = new(); + public int[] SecondaryColumnMap { get; init; } = Array.Empty(); } public static class TableMatcher @@ -296,7 +297,7 @@ public static class TableMatcher private static bool IsEmptyParts(string[] parts) => parts.All(string.IsNullOrEmpty); - private static string JoinKey(string[] parts) => string.Join("\u00a6", parts); + private static string JoinKey(string[] parts) => string.Concat(parts.Select(part => $"{part.Length}:{part};")); private static bool PartsContain(string[] a, string[] b) { @@ -360,21 +361,24 @@ public static class TableMatcher private sealed class Candidate { - public string[]? BRow; + public List<(string[] Row, double Score)> BRows = new(); public string Type = ""; - public double Score; - public int BCount; } private static List MatchMain(TableData main, TableData secondary, MatchOptions o, out List onlySecondary) { var mainIdx = KeyIndexes(o.MainUsed); var secIdx = KeyIndexes(o.SecondaryUsed); - int keyCount = Math.Min(mainIdx.Count, secIdx.Count); + if (mainIdx.Count != secIdx.Count) + throw new ArgumentException("Выберите одинаковое число ключевых колонок в обеих таблицах."); + int keyCount = mainIdx.Count; onlySecondary = new List(); if (keyCount == 0) - return main.Rows.Select(_ => (Candidate?)null).Cast().ToList(); + { + onlySecondary.AddRange(secondary.Rows); + return main.Rows.Select(_ => new Candidate()).ToList(); + } var token = o.Token; bool or = o.UseOr; @@ -388,34 +392,38 @@ public static class TableMatcher foreach (var row in main.Rows) { var parts = RowParts(row, mainIdx, keyCount, o); - if (!IsEmptyParts(parts)) mainKeys.Add(JoinKey(parts)); + if (parts.All(part => !string.IsNullOrEmpty(part))) mainKeys.Add(JoinKey(parts)); } - var bByKey = new Dictionary(StringComparer.Ordinal); + var bByKey = new Dictionary>(StringComparer.Ordinal); foreach (var row in secondary.Rows) { token.ThrowIfCancellationRequested(); var parts = RowParts(row, secIdx, keyCount, o); - if (IsEmptyParts(parts)) continue; + if (parts.Any(string.IsNullOrEmpty)) continue; var k = JoinKey(parts); - if (bByKey.TryGetValue(k, out var cur)) - bByKey[k] = (cur.Row, cur.Count + 1); + if (bByKey.TryGetValue(k, out var rows)) + rows.Add(row); else - bByKey[k] = (row, 1); + bByKey[k] = new List { row }; } foreach (var row in main.Rows) { token.ThrowIfCancellationRequested(); var parts = RowParts(row, mainIdx, keyCount, o); - if (IsEmptyParts(parts)) + if (parts.Any(string.IsNullOrEmpty)) { candidates.Add(new Candidate()); continue; } if (bByKey.TryGetValue(JoinKey(parts), out var hit)) { - candidates.Add(new Candidate { BRow = hit.Row, Type = "точное", Score = 100, BCount = hit.Count }); + candidates.Add(new Candidate + { + Type = "точное", + BRows = hit.Select(row => (row, 100.0)).ToList() + }); } else { @@ -427,7 +435,7 @@ public static class TableMatcher { token.ThrowIfCancellationRequested(); var parts = RowParts(row, secIdx, keyCount, o); - if (!IsEmptyParts(parts) && !mainKeys.Contains(JoinKey(parts))) + if (parts.Any(string.IsNullOrEmpty) || !mainKeys.Contains(JoinKey(parts))) onlySecondary.Add(row); } } @@ -443,21 +451,21 @@ public static class TableMatcher if (parts[j].Length > 0) mainSets[j].Add(parts[j]); } - var bByVal = new Dictionary[keyCount]; - for (int j = 0; j < keyCount; j++) bByVal[j] = new Dictionary(StringComparer.Ordinal); - foreach (var row in secondary.Rows) + var bByVal = new Dictionary>[keyCount]; + for (int j = 0; j < keyCount; j++) bByVal[j] = new Dictionary>(StringComparer.Ordinal); + for (int rowIndex = 0; rowIndex < secondary.Rows.Count; rowIndex++) { token.ThrowIfCancellationRequested(); - var parts = RowParts(row, secIdx, keyCount, o); + var parts = RowParts(secondary.Rows[rowIndex], secIdx, keyCount, o); if (IsEmptyParts(parts)) continue; for (int j = 0; j < keyCount; j++) { if (parts[j].Length == 0) continue; var v = parts[j]; - if (bByVal[j].TryGetValue(v, out var cur)) - bByVal[j][v] = (cur.Row, cur.Count + 1); + if (bByVal[j].TryGetValue(v, out var rowIndexes)) + rowIndexes.Add(rowIndex); else - bByVal[j][v] = (row, 1); + bByVal[j][v] = new List { rowIndex }; } } @@ -470,28 +478,28 @@ public static class TableMatcher candidates.Add(new Candidate()); continue; } - string[]? bestRow = null; - int bestCount = 0; + var matchedRows = new HashSet(); for (int j = 0; j < keyCount; j++) { if (parts[j].Length == 0) continue; - if (bByVal[j].TryGetValue(parts[j], out var hit) && hit.Count > bestCount) - { - bestRow = hit.Row; - bestCount = hit.Count; - } + if (bByVal[j].TryGetValue(parts[j], out var rowIndexes)) + foreach (var rowIndex in rowIndexes) matchedRows.Add(rowIndex); } - if (bestRow is not null) - candidates.Add(new Candidate { BRow = bestRow, Type = "точное", Score = 100, BCount = bestCount }); - else - candidates.Add(new Candidate()); + var candidate = new Candidate { Type = "точное" }; + foreach (var rowIndex in matchedRows.OrderBy(index => index)) + candidate.BRows.Add((secondary.Rows[rowIndex], 100.0)); + candidates.Add(candidate); } foreach (var row in secondary.Rows) { token.ThrowIfCancellationRequested(); var parts = RowParts(row, secIdx, keyCount, o); - if (IsEmptyParts(parts)) continue; + if (IsEmptyParts(parts)) + { + onlySecondary.Add(row); + continue; + } bool matched = false; for (int j = 0; j < keyCount; j++) { @@ -518,15 +526,13 @@ public static class TableMatcher { token.ThrowIfCancellationRequested(); var parts = RowParts(row, mainIdx, keyCount, o); - if (IsEmptyParts(parts)) + if (IsEmptyParts(parts) || (!or && parts.Any(string.IsNullOrEmpty))) { candidates.Add(new Candidate()); continue; } - string[]? bestRow = null; - double bestScore = 0; - int hitCount = 0; + var candidate = new Candidate { Type = o.Kind == MatchKind.Partial ? "частичное" : "нечёткое" }; if (o.Kind == MatchKind.Partial) { @@ -536,13 +542,9 @@ public static class TableMatcher bool ok = or ? AnyColContains(parts, b.Parts) : PartsContain(parts, b.Parts); if (!ok) continue; hits[i] = true; - hitCount++; - if (bestRow is null) bestRow = b.Row; + candidate.BRows.Add((b.Row, 100.0)); } - if (bestRow is not null) - candidates.Add(new Candidate { BRow = bestRow, BCount = hitCount, Score = 100, Type = "частичное" }); - else - candidates.Add(new Candidate()); + candidates.Add(candidate); } else { @@ -550,28 +552,15 @@ public static class TableMatcher for (int i = 0; i < secRows.Count; i++) { var b = secRows[i]; + if (!or && b.Parts.Any(string.IsNullOrEmpty)) continue; double sim = ColumnSim(parts, b.Parts, or); if (sim >= threshold) { hits[i] = true; - hitCount++; - if (sim > bestScore) - { - bestScore = sim; - bestRow = b.Row; - } + candidate.BRows.Add((b.Row, Math.Round(sim * 100, 1))); } } - if (bestRow is not null) - candidates.Add(new Candidate - { - BRow = bestRow, - BCount = hitCount, - Score = Math.Round(bestScore * 100, 1), - Type = "нечёткое" - }); - else - candidates.Add(new Candidate()); + candidates.Add(candidate); } } @@ -596,20 +585,23 @@ public static class TableMatcher { o.Token.ThrowIfCancellationRequested(); var c = candidates[i]; - if (c.BRow is null) + if (c.BRows.Count == 0) { onlyMain.Add(main.Rows[i]); continue; } matches.Add(main.Rows[i]); - pairs.Add(new MatchPair + foreach (var matched in c.BRows) { - RowA = main.Rows[i], - RowB = c.BRow, - Type = c.Type, - Score = c.Score, - BCount = c.BCount - }); + pairs.Add(new MatchPair + { + RowA = main.Rows[i], + RowB = matched.Row, + Type = c.Type, + Score = matched.Score, + BCount = c.BRows.Count + }); + } } return new MatchResult @@ -633,65 +625,98 @@ public static class TableMatcher Kind = partial ? MatchKind.Partial : MatchKind.Exact }); - public static MergeOutput Merge(TableData main, TableData secondary, MatchOptions o, bool removeIdenticalColumns = true) + public static MergeOutput Merge(TableData main, TableData secondary, MatchOptions o, bool mergeColumnsByHeader = true) { var candidates = MatchMain(main, secondary, o, out _); - var columns = new List(main.Columns); - var extraCols = new List(); + var secondaryColumnMap = new int[secondary.Columns.Count]; + var commonMainColumn = Enumerable.Repeat(-1, secondary.Columns.Count).ToArray(); + var usedMainColumns = new HashSet(); + var conflictingSecondaryColumns = new bool[secondary.Columns.Count]; - if (removeIdenticalColumns) + if (mergeColumnsByHeader) { for (int j = 0; j < secondary.Columns.Count; j++) { - if (columns.Contains(secondary.Columns[j])) - continue; - - var target = SortedColumnValues(secondary.Rows, j); - bool duplicate = false; - for (int c = 0; c < main.Columns.Count && !duplicate; c++) - duplicate = ColumnValuesEqual(target, SortedColumnValues(main.Rows, c)); - for (int e = 0; e < extraCols.Count && !duplicate; e++) - duplicate = ColumnValuesEqual(target, SortedColumnValues(secondary.Rows, extraCols[e])); - if (duplicate) continue; - - columns.Add(secondary.Columns[j]); - extraCols.Add(j); - } - } - else - { - for (int j = 0; j < secondary.Columns.Count; j++) - { - columns.Add(secondary.Columns[j]); - extraCols.Add(j); + commonMainColumn[j] = Enumerable.Range(0, main.Columns.Count) + .FirstOrDefault(c => !usedMainColumns.Contains(c) && + string.Equals(main.Columns[c].Trim(), secondary.Columns[j].Trim(), StringComparison.OrdinalIgnoreCase), -1); + if (commonMainColumn[j] >= 0) usedMainColumns.Add(commonMainColumn[j]); } } - string[] BuildMerged(string[] row, string[] secRow) + // If same-named fields contain different non-empty values in any matched + // pair, keep the B value in its own column instead of silently discarding it. + for (int i = 0; i < candidates.Count && i < main.RowCount; i++) { - var merged = new string[columns.Count]; - for (int c = 0; c < main.Columns.Count; c++) - merged[c] = c < row.Length ? row[c] : ""; - for (int e = 0; e < extraCols.Count; e++) + foreach (var match in candidates[i].BRows) { - int j = extraCols[e]; - merged[main.Columns.Count + e] = j < secRow.Length ? secRow[j] : ""; + for (int j = 0; j < secondary.Columns.Count; j++) + { + int a = commonMainColumn[j]; + if (a < 0) continue; + string av = a < main.Rows[i].Length ? main.Rows[i][a] : ""; + string bv = j < match.Row.Length ? match.Row[j] : ""; + if (!string.IsNullOrWhiteSpace(av) && !string.IsNullOrWhiteSpace(bv) && + !string.Equals(av, bv, StringComparison.Ordinal)) + conflictingSecondaryColumns[j] = true; + } } - return merged; + } + + int AddUniqueColumn(string header) + { + string baseHeader = string.IsNullOrWhiteSpace(header) ? "Колонка B" : header; + string unique = baseHeader; + int suffix = 2; + while (columns.Contains(unique, StringComparer.OrdinalIgnoreCase)) + unique = $"{baseHeader} ({suffix++})"; + columns.Add(unique); + return columns.Count - 1; + } + + for (int j = 0; j < secondary.Columns.Count; j++) + { + if (mergeColumnsByHeader && commonMainColumn[j] >= 0 && !conflictingSecondaryColumns[j]) + { + secondaryColumnMap[j] = commonMainColumn[j]; + continue; + } + + string header = secondary.Columns[j]; + if (!mergeColumnsByHeader || commonMainColumn[j] >= 0 || columns.Contains(header, StringComparer.OrdinalIgnoreCase)) + header = "B: " + header; + secondaryColumnMap[j] = AddUniqueColumn(header); } var rows = new List(); - int n = Math.Min(candidates.Count, main.RowCount); - for (int i = 0; i < n; i++) + for (int i = 0; i < candidates.Count && i < main.RowCount; i++) { o.Token.ThrowIfCancellationRequested(); - var c = candidates[i]; - if (c.BRow is not null) - rows.Add(BuildMerged(main.Rows[i], c.BRow)); + foreach (var match in candidates[i].BRows) + { + var merged = new string[columns.Count]; + Array.Fill(merged, ""); + for (int c = 0; c < main.Columns.Count; c++) + if (c < main.Rows[i].Length) merged[c] = main.Rows[i][c]; + for (int j = 0; j < secondaryColumnMap.Length; j++) + { + string bv = j < match.Row.Length ? match.Row[j] : ""; + int destination = secondaryColumnMap[j]; + if (commonMainColumn[j] >= 0 && !conflictingSecondaryColumns[j]) + { + if (string.IsNullOrWhiteSpace(merged[destination])) merged[destination] = bv; + } + else + { + merged[destination] = bv; + } + } + rows.Add(merged); + } } - return new MergeOutput { Columns = columns, Rows = rows }; + return new MergeOutput { Columns = columns, Rows = rows, SecondaryColumnMap = secondaryColumnMap }; } public static MergeOutput Merge( @@ -701,51 +726,26 @@ public static class TableMatcher bool[] secondaryUsed, bool ignoreCaseAndSpaces, bool partial = false, - bool removeIdenticalColumns = true) + bool mergeColumnsByHeader = true) => Merge(main, secondary, new MatchOptions { MainUsed = mainUsed, SecondaryUsed = secondaryUsed, IgnoreCaseAndSpaces = ignoreCaseAndSpaces, Kind = partial ? MatchKind.Partial : MatchKind.Exact - }, removeIdenticalColumns); - - private static List SortedColumnValues(List rows, int colIndex) - { - var vals = new List(); - foreach (var r in rows) - vals.Add(colIndex < r.Length ? (r[colIndex] ?? "").Trim() : ""); - vals.Sort(StringComparer.Ordinal); - return vals; - } - - private static bool ColumnValuesEqual(List a, List b) - { - if (a.Count != b.Count) return false; - for (int i = 0; i < a.Count; i++) - if (!string.Equals(a[i], b[i], StringComparison.Ordinal)) return false; - return true; - } + }, mergeColumnsByHeader); public static (TableData Table, int Removed) Dedupe(TableData table, bool[] used, bool ignore, MatchOptions? template = null) { - var idx = KeyIndexes(used); - if (idx.Count == 0) - return (table, 0); - var o = template?.Clone() ?? new MatchOptions(); - o.MainUsed = used; - o.SecondaryUsed = used; - o.IgnoreCaseAndSpaces = ignore; - var seen = new HashSet(StringComparer.Ordinal); var rows = new List(); int removed = 0; foreach (var row in table.Rows) { o.Token.ThrowIfCancellationRequested(); - var parts = RowParts(row, idx, idx.Count, o); - if (!IsEmptyParts(parts) && !seen.Add(JoinKey(parts))) + var parts = row.Select(value => value ?? "").ToArray(); + if (!seen.Add(JoinKey(parts))) { removed++; continue; @@ -849,9 +849,10 @@ public static class ExcelExporter } if (withMeta) { - var type = result.Pairs.Count > i ? result.Pairs[i].Type : ""; - var score = result.Pairs.Count > i ? result.Pairs[i].Score : 0; - var count = result.Pairs.Count > i ? result.Pairs[i].BCount : 0; + var rowPairs = result.Pairs.Where(p => ReferenceEquals(p.RowA, row)).ToList(); + var type = rowPairs.FirstOrDefault()?.Type ?? ""; + var score = rowPairs.Count == 0 ? 0 : rowPairs.Max(p => p.Score); + var count = rowPairs.Count; SetCell(ws, i + 2, baseCols + 1, type); SetCell(ws, i + 2, baseCols + 2, type == "нечёткое" ? score.ToString("0.#", CultureInfo.InvariantCulture) : ""); SetCell(ws, i + 2, baseCols + 3, count == 0 ? "" : count.ToString(CultureInfo.InvariantCulture)); @@ -927,8 +928,9 @@ public static class ExcelExporter Row("Из них частичных", partial); Row("Из них нечётких", fuzzy); - double avg = result.Pairs.Count == 0 ? 0 : Math.Round(result.Pairs.Average(p => p.BCount), 2); - int maxBC = result.Pairs.Count == 0 ? 0 : result.Pairs.Max(p => p.BCount); + var countsPerMainRow = result.Pairs.GroupBy(p => p.RowA).Select(group => group.First().BCount).ToList(); + double avg = countsPerMainRow.Count == 0 ? 0 : Math.Round(countsPerMainRow.Average(), 2); + int maxBC = countsPerMainRow.Count == 0 ? 0 : countsPerMainRow.Max(); Row("Среднее B-совпадений (N→M)", avg); Row("Максимум B-совпадений (N→M)", maxBC); @@ -967,4 +969,4 @@ public static class ExcelExporter else cell.Value = value; } -} \ No newline at end of file +} diff --git a/MainWindow.cs b/MainWindow.cs index 25eeefa..0ae85e9 100644 --- a/MainWindow.cs +++ b/MainWindow.cs @@ -137,8 +137,8 @@ public sealed class MainWindow : Window private readonly RadioButton _rbBoth = new() { Content = "Совпадения + чего нет в A" }; private readonly RadioButton _rbExact = new() { Content = "Точное совпадение", IsChecked = true }; private readonly RadioButton _rbPartial = new() { Content = "Частичное (по фрагменту)" }; - private readonly CheckBox _chkDedupe = new() { Content = "Удалять дубликаты по ключевым колонкам" }; - private readonly CheckBox _chkCleanCols = new() { Content = "Убирать одинаковые колонки после слияния", IsChecked = true }; + private readonly CheckBox _chkDedupe = new() { Content = "Удалять полностью одинаковые строки" }; + private readonly CheckBox _chkCleanCols = new() { Content = "Объединять одноимённые колонки", IsChecked = true }; private readonly RadioButton _rbFuzzy = new() { Content = "Нечёткое (по сходству %)" }; private readonly Slider _slFuzzy = new() @@ -561,7 +561,7 @@ private Control BuildFilePanel(TextBox tb, Button browse, ComboBox cb, StackPane var colLabel = new TextBlock { - Text = "Ключевые колонки для сопоставления", + Text = "Ключи (выберите соответствующие поля в одинаковом порядке)", Foreground = Ui.Muted, FontWeight = FontWeight.SemiBold, FontSize = 12 @@ -900,6 +900,11 @@ private Control BuildFilePanel(TextBox tb, Button browse, ComboBox cb, StackPane ShowError("Отметьте хотя бы одну ключевую колонку в каждой таблице (по умолчанию отмечена первая)."); return; } + if (mainUsed.Count(x => x) != secUsed.Count(x => x)) + { + ShowError("Выберите одинаковое число ключевых колонок в обеих таблицах."); + return; + } var cts = new CancellationTokenSource(); _cts = cts; @@ -978,8 +983,9 @@ private Control BuildFilePanel(TextBox tb, Button browse, ComboBox cb, StackPane sb.Append($"Совпадений: {r.Matches.Count} · Только в A: {r.OnlyMain.Count} · Только в B: {r.OnlySecondary.Count}."); if (r.Pairs.Count > 0) { - double avg = Math.Round(r.Pairs.Average(p => p.BCount), 2); - int max = r.Pairs.Max(p => p.BCount); + var countsPerMainRow = r.Pairs.GroupBy(p => p.RowA).Select(group => group.First().BCount).ToList(); + double avg = Math.Round(countsPerMainRow.Average(), 2); + int max = countsPerMainRow.Max(); sb.Append($" Пары A↔B: {r.Pairs.Count}, ср. N→M: {avg.ToString(CultureInfo.InvariantCulture)}, макс: {max}."); } sb.Append($" Ключи заполнены: A={covA.ToString(CultureInfo.InvariantCulture)}%, B={covB.ToString(CultureInfo.InvariantCulture)}%."); @@ -1081,59 +1087,27 @@ private Control BuildFilePanel(TextBox tb, Button browse, ComboBox cb, StackPane break; default: - // Build a union of both schemas. Columns with the same header are - // aligned; B-only columns get their own position in the result. - headers = new List(main.Columns); - var mainColumnMap = Enumerable.Range(0, main.Columns.Count).ToArray(); - var secondaryColumnMap = new int[sec.Columns.Count]; - var matchedSecondaryColumns = new HashSet(); - - for (int b = 0; b < sec.Columns.Count; b++) - { - int matchingMain = -1; - for (int a = 0; a < main.Columns.Count; a++) - { - if (!matchedSecondaryColumns.Contains(a) && - string.Equals(main.Columns[a].Trim(), sec.Columns[b].Trim(), StringComparison.OrdinalIgnoreCase)) - { - matchingMain = a; - break; - } - } - - if (matchingMain >= 0) - { - secondaryColumnMap[b] = matchingMain; - matchedSecondaryColumns.Add(matchingMain); - } - else - { - var header = sec.Columns[b]; - if (headers.Contains(header, StringComparer.OrdinalIgnoreCase)) - header = $"B: {header}"; - secondaryColumnMap[b] = headers.Count; - headers.Add(header); - } - } - - string[] CombineRows(string[] rowA, string[] rowB) - { - var combined = new string[headers.Count]; - Array.Fill(combined, ""); - for (int a = 0; a < mainColumnMap.Length && a < rowA.Length; a++) - combined[mainColumnMap[a]] = rowA[a]; - for (int b = 0; b < secondaryColumnMap.Length && b < rowB.Length; b++) - combined[secondaryColumnMap[b]] = rowB[b]; - return combined; - } - - foreach (var pair in result.Pairs) - data.Add(CombineRows(pair.RowA, pair.RowB)); + headers = _merged?.Columns ?? new List(main.Columns); + data = _merged is not null ? new List(_merged.Rows) : new List(); sources = new List(); foreach (var _ in result.Pairs) sources.Add("A+B"); foreach (var r in result.OnlySecondary) { - data.Add(CombineRows(Array.Empty(), r)); + var row = new string[headers.Count]; + Array.Fill(row, ""); + if (_merged is not null) + { + for (int b = 0; b < _merged.SecondaryColumnMap.Length && b < r.Length; b++) + { + int destination = _merged.SecondaryColumnMap[b]; + if (destination >= 0 && destination < row.Length) row[destination] = r[b]; + } + } + else + { + for (int b = 0; b < r.Length && b < row.Length; b++) row[b] = r[b]; + } + data.Add(row); sources.Add("B"); } break;