From 3f133f117c6d5a845c293065e86cef24c5352d68 Mon Sep 17 00:00:00 2001 From: Dan Siroky Date: Mon, 17 Mar 2025 16:17:50 -0500 Subject: [PATCH 1/9] Add tests project --- HOK.Elastic.Tests/HOK.Elastic.Tests.csproj | 22 +++ HOK.Elastic.Tests/HOK.Elastic.Tests.sln | 24 +++ HOK.Elastic.Tests/LocalCluster.cs | 22 +++ HOK.Elastic.Tests/TestUtils.cs | 23 +++ HOK.Elastic.Tests/Tests.cs | 205 +++++++++++++++++++++ HOK.Elastic.sln | 10 + 6 files changed, 306 insertions(+) create mode 100644 HOK.Elastic.Tests/HOK.Elastic.Tests.csproj create mode 100644 HOK.Elastic.Tests/HOK.Elastic.Tests.sln create mode 100644 HOK.Elastic.Tests/LocalCluster.cs create mode 100644 HOK.Elastic.Tests/TestUtils.cs create mode 100644 HOK.Elastic.Tests/Tests.cs diff --git a/HOK.Elastic.Tests/HOK.Elastic.Tests.csproj b/HOK.Elastic.Tests/HOK.Elastic.Tests.csproj new file mode 100644 index 0000000..62514bd --- /dev/null +++ b/HOK.Elastic.Tests/HOK.Elastic.Tests.csproj @@ -0,0 +1,22 @@ + + + net8.0-windows + enable + enable + + + + + + + + + + + + + + + + + diff --git a/HOK.Elastic.Tests/HOK.Elastic.Tests.sln b/HOK.Elastic.Tests/HOK.Elastic.Tests.sln new file mode 100644 index 0000000..7837958 --- /dev/null +++ b/HOK.Elastic.Tests/HOK.Elastic.Tests.sln @@ -0,0 +1,24 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.2.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HOK.Elastic.Tests", "HOK.Elastic.Tests.csproj", "{9D1A765D-0600-5F8C-4676-855FB237CE1F}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {9D1A765D-0600-5F8C-4676-855FB237CE1F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9D1A765D-0600-5F8C-4676-855FB237CE1F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9D1A765D-0600-5F8C-4676-855FB237CE1F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9D1A765D-0600-5F8C-4676-855FB237CE1F}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {FD065F66-79D3-4308-9C3E-E63672F67936} + EndGlobalSection +EndGlobal diff --git a/HOK.Elastic.Tests/LocalCluster.cs b/HOK.Elastic.Tests/LocalCluster.cs new file mode 100644 index 0000000..436a97f --- /dev/null +++ b/HOK.Elastic.Tests/LocalCluster.cs @@ -0,0 +1,22 @@ +using Elastic.Elasticsearch.Xunit; +using Elastic.Elasticsearch.Ephemeral; +using Elastic.Elasticsearch.Ephemeral.Plugins; +using Elastic.Stack.ArtifactsApi.Products; + +namespace HOK.Elastic.Tests +{ + /// Declare our cluster that we want to inject into our test classes + public class LocalCluster : XunitClusterBase + { + /// + /// We pass our configuration instance to the base class. + /// + public LocalCluster() : base(new XunitClusterConfiguration( + "8.17.2", + ClusterFeatures.None, + new ElasticsearchPlugins(new List {}), + 1 + )) + { } + } +} diff --git a/HOK.Elastic.Tests/TestUtils.cs b/HOK.Elastic.Tests/TestUtils.cs new file mode 100644 index 0000000..65855e8 --- /dev/null +++ b/HOK.Elastic.Tests/TestUtils.cs @@ -0,0 +1,23 @@ +namespace HOK.Elastic.Tests; + +public class TestUtils +{ + public static void CopyFilesRecursively(string sourcePath, string targetPath) + { + //Now Create all of the directories + foreach ( + string dirPath in Directory.GetDirectories(sourcePath, "*", SearchOption.AllDirectories) + ) + { + Directory.CreateDirectory(dirPath.Replace(sourcePath, targetPath)); + } + + //Copy all the files & Replaces any files with the same name + foreach ( + string newPath in Directory.GetFiles(sourcePath, "*.*", SearchOption.AllDirectories) + ) + { + File.Copy(newPath, newPath.Replace(sourcePath, targetPath), true); + } + } +} diff --git a/HOK.Elastic.Tests/Tests.cs b/HOK.Elastic.Tests/Tests.cs new file mode 100644 index 0000000..06c5fb2 --- /dev/null +++ b/HOK.Elastic.Tests/Tests.cs @@ -0,0 +1,205 @@ +using FluentAssertions; +using HOK.Elastic.DAL; +using HOK.Elastic.DAL.Models; +using HOK.Elastic.FileSystemCrawler; +using HOK.Elastic.FileSystemCrawler.Models; +using Microsoft.Extensions.Logging; +using NLog.Extensions.Logging; +using Xunit; + +namespace HOK.Elastic.Tests; + +public class FileSystemCrawlerFixture : IDisposable +{ + public FileSystemCrawlerFixture() + { + // called once before all tests + string inputPath = @"C:\Elastic\HOK.Elastic Tests\Template\Standard"; + string workingPathBase = @"C:\Temp"; + string workingPathUnc = @$"\\{System.Environment.MachineName.ToLowerInvariant()}\c$\Temp"; + string office = "abc"; + tempPath = @$"{workingPathBase}\{office}"; + + var uri = "https://elastic-dev-cluster:9200"; + var workerargs = new SettingsJobArgs() + { + // Things to input + ElasticDiscoveryURI = new List { new Uri(uri) }, + ElasticIndexURI = new List { new Uri(uri) }, + //ReadFileContents = jobSettings.ReadFileContents, + //PathForCrawling = jobSettings.PathForCrawling, + //PathForCrawlingContent = jobSettings.PathForCrawlingContent, + //PublishedPath = jobSettings.PublishedPath, + + DocInsertionThreads = 1, + CPUCoreThreadMultiplier = 1, + ReadContentSizeLimitMB = 200, + BulkUploadSize = 500, + IgnoreExtensions = new List(), + OfficeSiteExtractRegex = $"^{workingPathBase}".Replace("\\", "\\\\") + @"(\w{2,4})", + ProjectExtractRegex = + $"^{workingPathBase}".Replace("\\", "\\\\") + + @"\w{2,4}\\projects\\((\d\d\d\d\\(\d{2}[\d|\.\-]*)+\s*(\+|_|\-)?\s*([^\\|$|\r|\n]*)|(interiors\\|planning\\|architecture\\|hospitality\\)?([^\\]*)?))", + PipeCategorizationRegex = + $"^{workingPathBase}".Replace("\\", "\\\\") + + @"(\w{2,4})\\projects\\.*?\\[a-z]\s?\-\s?(?.*?)\\", + // Don't Set + //PathInclusionRegex = jobSettings.PathInclusionRegex ?? AppSettings.PathInclusionRegex, + //FileNameExclusionRegex = jobSettings.FileNameExclusionRegex ?? AppSettings.FileNameExclusionRegex, + //CrawlMode = jobSettings.CrawlMode, (will do in the test itself) + //InputPathLocation = jobDirectoryInfo.FullName, + + ExceptionsPerTenMinuteIntervalLimit = 100 + }; + DateTime date = DateTime.Now; + var shortDate = date.ToString("yyyyMMdd"); + workerargs.IndexNamePrefix = $"testrrunner-{shortDate}."; + + var workingPath = + $@"{workingPathBase}\{office}\projects\2025\25.12345.00 Crawl Test Project\"; + TestUtils.CopyFilesRecursively(inputPath, workingPath); + workingPath = + $@"{workingPathUnc}\{office}\projects\2025\25.12345.00 Crawl Test Project\"; + var inputPathBase = new InputPathCollectionBase + { + new InputPathBase(workingPath, office, PathStatus.Unstarted) + }; + inputPathBase.PathForCrawling = workingPathUnc; + inputPathBase.PathForCrawlingContent = workingPathBase; + inputPathBase.PublishedPath = workingPathBase; + workerargs.InputPaths = inputPathBase; + workerargs.PathForCrawling = workingPathUnc; + workerargs.PathForCrawlingContent = workingPathUnc; + workerargs.PublishedPath = workingPathUnc; + workerargs.InputPathLocation = workingPathBase; + workerargs.RunningInteractively = false; + PathHelper.Set(workerargs.PublishedPath, workerargs.PathForCrawlingContent, workerargs.PathForCrawling); + + PathHelper.SetPathInclusion(workerargs.PathInclusionRegex); + PathHelper.SetFileNameExclusion(workerargs.FileNameExclusionRegex); + PathHelper.SetOfficeExtractRgx(workerargs.OfficeSiteExtractRegex); + PathHelper.SetProjectExtractRgx(workerargs.ProjectExtractRegex); + PathHelper.IgnoreExtensions = workerargs.IgnoreExtensions?.Distinct().ToHashSet(); + //fileSystemCrawlTest = new FileSystemCrawlTest(workerargs); + settingsJobArgs = workerargs; + _loggerFactory = LoggerFactory.Create(builder => builder.AddNLog()); + IndexNameHelper indexNameHelper = new IndexNameHelper(workerargs.IndexNamePrefix); + PipeLineNameHelper pipeLineNameHelper = new PipeLineNameHelper( + workerargs.IndexNamePrefix + ); + + discovery = new DAL.Discovery( + pipeLineNameHelper, + indexNameHelper, + workerargs.ElasticDiscoveryURI.First(), + _loggerFactory.CreateLogger($"{workerargs.JobName}.Discovery") + ); + index = new DAL.Index( + pipeLineNameHelper, + indexNameHelper, + workerargs.ElasticIndexURI.First(), + _loggerFactory.CreateLogger($"{workerargs.JobName}.Index") + ); + securityHelper = new SecurityHelper( + _loggerFactory.CreateLogger($"{workerargs.JobName}.SecurityHelper") + ); + documentHelper = new DocumentHelper( + workerargs.ReadFileContents ?? false, + securityHelper, + index, + _loggerFactory.CreateLogger($"{workerargs.JobName}.DocumentHelper") + ); + + worker = new WorkerCrawler( + index, + discovery, + securityHelper, + documentHelper, + _loggerFactory.CreateLogger($"{workerargs.JobName}.WorkerCrawler") + ); + + PipeLineNameHelper pipeLineHelper = new PipeLineNameHelper(workerargs.IndexNamePrefix); + + using (var initializationPipeline = new InitializationPipeline(pipeLineNameHelper, indexNameHelper, workerargs.ElasticIndexURI.First(), _loggerFactory.CreateLogger($"{workerargs.JobName}.Setup"))) + { + if (!initializationPipeline.CheckForPipeLines()) + { + initializationPipeline.Put(true); + } + + using (var initializationIndex = new InitializationIndex(pipeLineNameHelper, indexNameHelper, workerargs.ElasticIndexURI.First(), _loggerFactory.CreateLogger($"{workerargs.JobName}.IndexSetup"))) + { + if (initializationIndex.PreFlightFail()) + { + initializationIndex.Put(workerargs.RunningInteractively); + } + } + + } + } + + + public void Dispose() + { + // called once after all tests + //File.Delete(tempPath); + } + + public string tempPath { get; private set; } + public SettingsJobArgs settingsJobArgs { get; private set; } + public CompletionInfo? completionInfo; + private readonly ILoggerFactory _loggerFactory; + public DAL.Discovery discovery { get; private set; } + private DAL.Index index { get; set; } + private SecurityHelper securityHelper { get; set; } + private DocumentHelper documentHelper { get; set; } + public WorkerCrawler worker { get; private set; } +} + +public class FilesystemCrawlTest: IClassFixture { + public FileSystemCrawlerFixture fixture; + public FilesystemCrawlTest(FileSystemCrawlerFixture fixture) + { + this.fixture = fixture; + } + + [Fact] + public async void FullCrawlTestSuite() + { + CancellationTokenSource _ct = new CancellationTokenSource(); + + //**Full crawl** + fixture.settingsJobArgs.CrawlMode = CrawlMode.Full; + CompletionInfo completionInfo = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + + var path = fixture.worker.GetDirectoriesFromInputPaths(new List { fixture.settingsJobArgs.InputPaths.First() }, fixture.settingsJobArgs).First(); + //var path = fixture.settingsJobArgs.InputPaths.First(); + if (path == null) + throw new Exception("Path is not defined"); + + // TODO: This path doesn't match what's in Elastic + var crawlContents = fixture.discovery.FindRootAndChildren(path.PublishedPath, true); + + //-Check doc count, broken down by index + var totalDocCount = crawlContents.Contents.Count(); + var dirCount = crawlContents.Contents.Count( + x => x.Item2 == $"{fixture.settingsJobArgs.IndexNamePrefix}dir" + ); + + dirCount.Should().BeGreaterOrEqualTo(1); + + //-Check properties (metadata) + //Change permissions + //Delete a file + //**Incremental crawl** + //-Check doc count, broken down by index + //-Check doc count as different user with less permissions??? + //-Check properties (metadata) + //Add a file + //**Incremental crawl** + //-Check doc count, broken down by index + //-Check doc count as different user with less permissions??? + //-Check properties (metadata) + + } +} \ No newline at end of file diff --git a/HOK.Elastic.sln b/HOK.Elastic.sln index f510b95..f0c4156 100644 --- a/HOK.Elastic.sln +++ b/HOK.Elastic.sln @@ -18,6 +18,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HOK.Elastic.FileSystemCrawl EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HOK.Elastic.ArchiveDiscovery", "HOK.Elastic.ArchiveDiscovery\HOK.Elastic.ArchiveDiscovery.csproj", "{7633149C-6D28-4CAE-B931-CD1080092ACD}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HOK.Elastic.Tests", "HOK.Elastic.Tests\HOK.Elastic.Tests.csproj", "{EF90D6C1-F8EC-45FE-A7C7-3527865F413F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -90,6 +92,14 @@ Global {7633149C-6D28-4CAE-B931-CD1080092ACD}.Release|Any CPU.Build.0 = Release|Any CPU {7633149C-6D28-4CAE-B931-CD1080092ACD}.Release|x64.ActiveCfg = Release|Any CPU {7633149C-6D28-4CAE-B931-CD1080092ACD}.Release|x64.Build.0 = Release|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Debug|x64.ActiveCfg = Debug|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Debug|x64.Build.0 = Debug|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Release|Any CPU.Build.0 = Release|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Release|x64.ActiveCfg = Release|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 951bc232a3a834c071013ea606c82bd89128dab2 Mon Sep 17 00:00:00 2001 From: Dan Siroky Date: Mon, 17 Mar 2025 16:18:56 -0500 Subject: [PATCH 2/9] Update references based on vulnerability warnings --- .../HOK.Elastic.ArchiveDiscovery.csproj | 4 +-- HOK.Elastic.DAL/HOK.Elastic.DAL.csproj | 10 +++--- ...ic.FileSystemCrawler.ConsoleProgram.csproj | 34 +++++++++---------- ...OK.Elastic.FileSystemCrawler.WebAPI.csproj | 6 ++-- .../HOK.Elastic.FileSystemCrawler.csproj | 10 +++--- HOK.Elastic.Logger/HOK.Elastic.Logger.csproj | 4 +-- .../HOK.Elastic.RoleMappingGroupSync.csproj | 4 +-- .../HOK.NasuniAuditEventAPI.csproj | 6 ++-- 8 files changed, 39 insertions(+), 39 deletions(-) diff --git a/HOK.Elastic.ArchiveDiscovery/HOK.Elastic.ArchiveDiscovery.csproj b/HOK.Elastic.ArchiveDiscovery/HOK.Elastic.ArchiveDiscovery.csproj index 5c7388a..4c0e5f8 100644 --- a/HOK.Elastic.ArchiveDiscovery/HOK.Elastic.ArchiveDiscovery.csproj +++ b/HOK.Elastic.ArchiveDiscovery/HOK.Elastic.ArchiveDiscovery.csproj @@ -18,8 +18,8 @@ - - + + diff --git a/HOK.Elastic.DAL/HOK.Elastic.DAL.csproj b/HOK.Elastic.DAL/HOK.Elastic.DAL.csproj index 7987af4..7ac68a6 100644 --- a/HOK.Elastic.DAL/HOK.Elastic.DAL.csproj +++ b/HOK.Elastic.DAL/HOK.Elastic.DAL.csproj @@ -4,12 +4,12 @@ AnyCPU;x64 - - - - + + + + - + diff --git a/HOK.Elastic.FileSystemCrawler.ConsoleProgram/HOK.Elastic.FileSystemCrawler.ConsoleProgram.csproj b/HOK.Elastic.FileSystemCrawler.ConsoleProgram/HOK.Elastic.FileSystemCrawler.ConsoleProgram.csproj index 0ed6d78..1aca741 100644 --- a/HOK.Elastic.FileSystemCrawler.ConsoleProgram/HOK.Elastic.FileSystemCrawler.ConsoleProgram.csproj +++ b/HOK.Elastic.FileSystemCrawler.ConsoleProgram/HOK.Elastic.FileSystemCrawler.ConsoleProgram.csproj @@ -77,26 +77,26 @@ - - + + - - - - - - - - - - - - + + + + + + + + + + + + - + @@ -106,8 +106,8 @@ - - + + diff --git a/HOK.Elastic.FileSystemCrawler.WebAPI/HOK.Elastic.FileSystemCrawler.WebAPI.csproj b/HOK.Elastic.FileSystemCrawler.WebAPI/HOK.Elastic.FileSystemCrawler.WebAPI.csproj index ad284b6..6dbbfc9 100644 --- a/HOK.Elastic.FileSystemCrawler.WebAPI/HOK.Elastic.FileSystemCrawler.WebAPI.csproj +++ b/HOK.Elastic.FileSystemCrawler.WebAPI/HOK.Elastic.FileSystemCrawler.WebAPI.csproj @@ -11,9 +11,9 @@ - - - + + + diff --git a/HOK.Elastic.FileSystemCrawler/HOK.Elastic.FileSystemCrawler.csproj b/HOK.Elastic.FileSystemCrawler/HOK.Elastic.FileSystemCrawler.csproj index 472e406..fefe79b 100644 --- a/HOK.Elastic.FileSystemCrawler/HOK.Elastic.FileSystemCrawler.csproj +++ b/HOK.Elastic.FileSystemCrawler/HOK.Elastic.FileSystemCrawler.csproj @@ -6,13 +6,13 @@ true - - + + - + - + @@ -23,7 +23,7 @@ - + diff --git a/HOK.Elastic.Logger/HOK.Elastic.Logger.csproj b/HOK.Elastic.Logger/HOK.Elastic.Logger.csproj index f02d67e..37ca6cb 100644 --- a/HOK.Elastic.Logger/HOK.Elastic.Logger.csproj +++ b/HOK.Elastic.Logger/HOK.Elastic.Logger.csproj @@ -4,8 +4,8 @@ AnyCPU;x64 - - + + diff --git a/HOK.Elastic.RoleMappingGroupSync/HOK.Elastic.RoleMappingGroupSync.csproj b/HOK.Elastic.RoleMappingGroupSync/HOK.Elastic.RoleMappingGroupSync.csproj index 4f2ce52..17b5dc7 100644 --- a/HOK.Elastic.RoleMappingGroupSync/HOK.Elastic.RoleMappingGroupSync.csproj +++ b/HOK.Elastic.RoleMappingGroupSync/HOK.Elastic.RoleMappingGroupSync.csproj @@ -5,8 +5,8 @@ AnyCPU;x64 - - + + diff --git a/HOK.NasuniAuditEventAPI/HOK.NasuniAuditEventAPI.csproj b/HOK.NasuniAuditEventAPI/HOK.NasuniAuditEventAPI.csproj index a2048f6..e0d1c0e 100644 --- a/HOK.NasuniAuditEventAPI/HOK.NasuniAuditEventAPI.csproj +++ b/HOK.NasuniAuditEventAPI/HOK.NasuniAuditEventAPI.csproj @@ -15,9 +15,9 @@ - - - + + + From 23c48a8a077509ab9320566762a274b49f97432d Mon Sep 17 00:00:00 2001 From: Dan Siroky Date: Mon, 17 Mar 2025 16:19:35 -0500 Subject: [PATCH 3/9] Changes to existing library based on adding test project --- HOK.Elastic.DAL/Discovery.cs | 2 -- HOK.Elastic.FileSystemCrawler.ConsoleProgram/Program.cs | 2 -- .../Models/InputPathCollectionEventStream.cs | 2 +- HOK.Elastic.FileSystemCrawler/WorkerBase.cs | 2 +- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/HOK.Elastic.DAL/Discovery.cs b/HOK.Elastic.DAL/Discovery.cs index 6a12cbb..6d0b6c7 100644 --- a/HOK.Elastic.DAL/Discovery.cs +++ b/HOK.Elastic.DAL/Discovery.cs @@ -65,7 +65,6 @@ public DirectoryContents FindRootAndChildren(string directoryPath, bool includeF int pageSize = 1000; int docCount = 0; - var lastCheck = 0; var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = 50 }; bool exit = false; bool firstTimePITUsed = true; @@ -614,7 +613,6 @@ public IEnumerable GetIFSOsByQuery(string jsonQueryString, int failureCoun /// public IEnumerable GetIFSOdocumentsLackingContentV2(string directoryPublishedPath, int failureCountFilter, DateTime? minimumDate = null) where T : class, IFSOdocument { - int counter = 0; long docCount = 0; PointInTimeDescriptor pointInTime = null; string pitID = null; diff --git a/HOK.Elastic.FileSystemCrawler.ConsoleProgram/Program.cs b/HOK.Elastic.FileSystemCrawler.ConsoleProgram/Program.cs index 5dd212d..008429b 100644 --- a/HOK.Elastic.FileSystemCrawler.ConsoleProgram/Program.cs +++ b/HOK.Elastic.FileSystemCrawler.ConsoleProgram/Program.cs @@ -20,8 +20,6 @@ using System.Threading.Tasks; using ILogger = Microsoft.Extensions.Logging.ILogger; using LogLevel = Microsoft.Extensions.Logging.LogLevel; -using HOK.Elastic.Logger; -using System.Collections.Generic; namespace HOK.Elastic.FileSystemCrawler.ConsoleProgram { diff --git a/HOK.Elastic.FileSystemCrawler/Models/InputPathCollectionEventStream.cs b/HOK.Elastic.FileSystemCrawler/Models/InputPathCollectionEventStream.cs index abec45f..1de7983 100644 --- a/HOK.Elastic.FileSystemCrawler/Models/InputPathCollectionEventStream.cs +++ b/HOK.Elastic.FileSystemCrawler/Models/InputPathCollectionEventStream.cs @@ -31,7 +31,7 @@ public void Add(InputPathEventStream[] newItems) } } } - public new void Add(InputPathEventStream newItem) + public void Add(InputPathEventStream newItem) { if (newItem == null) return; diff --git a/HOK.Elastic.FileSystemCrawler/WorkerBase.cs b/HOK.Elastic.FileSystemCrawler/WorkerBase.cs index 76e45b5..1b2c38b 100644 --- a/HOK.Elastic.FileSystemCrawler/WorkerBase.cs +++ b/HOK.Elastic.FileSystemCrawler/WorkerBase.cs @@ -66,7 +66,7 @@ public WorkerBase(IIndex elasticIngest, IDiscovery elasticDiscovery, DocumentHel public abstract CompletionInfo Run(ISettingsJobArgs args, CancellationToken ct = default); public abstract Task RunAsync(ISettingsJobArgs args, CancellationToken ct = default); - internal List GetDirectoriesFromInputPaths(List setofPaths, ISettingsJobArgs args)///TODO why not just use args.inputpaths? + public List GetDirectoriesFromInputPaths(List setofPaths, ISettingsJobArgs args)///TODO why not just use args.inputpaths? { var inputs = new List(); foreach (var item in setofPaths) From 558c9478f965339071aeb5765b35d3c8ada7d850 Mon Sep 17 00:00:00 2001 From: Dan Siroky Date: Mon, 17 Mar 2025 16:25:24 -0500 Subject: [PATCH 4/9] Rename Tests.cs to FileSystemCrawlTests.cs --- HOK.Elastic.Tests/{Tests.cs => FileSystemCrawlTests.cs} | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) rename HOK.Elastic.Tests/{Tests.cs => FileSystemCrawlTests.cs} (97%) diff --git a/HOK.Elastic.Tests/Tests.cs b/HOK.Elastic.Tests/FileSystemCrawlTests.cs similarity index 97% rename from HOK.Elastic.Tests/Tests.cs rename to HOK.Elastic.Tests/FileSystemCrawlTests.cs index 06c5fb2..a91b631 100644 --- a/HOK.Elastic.Tests/Tests.cs +++ b/HOK.Elastic.Tests/FileSystemCrawlTests.cs @@ -142,6 +142,7 @@ public FileSystemCrawlerFixture() public void Dispose() { // called once after all tests + // TODO: Cleanup tempPath and indices //File.Delete(tempPath); } @@ -156,9 +157,9 @@ public void Dispose() public WorkerCrawler worker { get; private set; } } -public class FilesystemCrawlTest: IClassFixture { +public class FileSystemCrawlTests : IClassFixture { public FileSystemCrawlerFixture fixture; - public FilesystemCrawlTest(FileSystemCrawlerFixture fixture) + public FileSystemCrawlTests(FileSystemCrawlerFixture fixture) { this.fixture = fixture; } From c7b3adce2d08780b75d399fc5edc899128a910ae Mon Sep 17 00:00:00 2001 From: jamesblackadar <33873820+jamesblackadar@users.noreply.github.com> Date: Wed, 26 Mar 2025 11:31:13 -0400 Subject: [PATCH 5/9] wip --- HOK.Elastic.DAL/Discovery.cs | 1 + HOK.Elastic.DAL/InitializationPipeline.cs | 7 +- .../Program.cs | 1 + .../WorkerCrawler.cs | 5 + HOK.Elastic.Tests/FileSystemCrawlTests.cs | 328 +++++++++++++++++- HOK.Elastic.Tests/TestUtils.cs | 39 ++- 6 files changed, 353 insertions(+), 28 deletions(-) diff --git a/HOK.Elastic.DAL/Discovery.cs b/HOK.Elastic.DAL/Discovery.cs index 6d0b6c7..71ccad8 100644 --- a/HOK.Elastic.DAL/Discovery.cs +++ b/HOK.Elastic.DAL/Discovery.cs @@ -131,6 +131,7 @@ public DirectoryContents FindRootAndChildren(string directoryPath, bool includeF catch (Exception ex) { if (ilerror) _il.LogErr($"Unexpected Error in {nameof(FindRootAndChildren)} because {ex.Message}", "",null,ex ); + exit = true; } } return directoryContents; diff --git a/HOK.Elastic.DAL/InitializationPipeline.cs b/HOK.Elastic.DAL/InitializationPipeline.cs index 0d10730..27e63a9 100644 --- a/HOK.Elastic.DAL/InitializationPipeline.cs +++ b/HOK.Elastic.DAL/InitializationPipeline.cs @@ -3,6 +3,7 @@ using Nest; using System; using System.Linq; +using System.Text.RegularExpressions; namespace HOK.Elastic.DAL { @@ -174,7 +175,8 @@ private PutPipelineResponse PutPipeValidator() /// example "^\\\\\\\\contoso\\\\projects\\\\.*?\\\\[a-z]\\s?\\-\\s?(?.*?)\\\\", /// private PutPipelineResponse PutPipeCategoryProject() - { + { + PutPipelineResponse response = client.Ingest .PutPipeline(PipeLineNameHelper.PIPECategorizationProject, p => p .Description("Pipeline to assign Category based on filepath.") @@ -182,7 +184,8 @@ private PutPipelineResponse PutPipeCategoryProject() .Grok(g=> g .Field(f=>f.Id) .PatternDefinitions(pd => pd.Add("CATEGORYPATTERN", PipeLineNameHelper.PIPECategorizationProjectExtractRgx)) - .Patterns("%{CATEGORYPATTERN:category}") + //.Patterns("%{CATEGORYPATTERN:category}") + .Patterns("%{CATEGORYPATTERN}") .IgnoreFailure(true) .IgnoreMissing(true) ) diff --git a/HOK.Elastic.FileSystemCrawler.ConsoleProgram/Program.cs b/HOK.Elastic.FileSystemCrawler.ConsoleProgram/Program.cs index 008429b..af17920 100644 --- a/HOK.Elastic.FileSystemCrawler.ConsoleProgram/Program.cs +++ b/HOK.Elastic.FileSystemCrawler.ConsoleProgram/Program.cs @@ -39,6 +39,7 @@ partial class Program //Main Entry Point static async Task Main(string[] args) { + ///not sure if this is actually needed here or in the msgreader library but it's all working currently... Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); int exitcode = 1; diff --git a/HOK.Elastic.FileSystemCrawler/WorkerCrawler.cs b/HOK.Elastic.FileSystemCrawler/WorkerCrawler.cs index 6b9031b..b05f982 100644 --- a/HOK.Elastic.FileSystemCrawler/WorkerCrawler.cs +++ b/HOK.Elastic.FileSystemCrawler/WorkerCrawler.cs @@ -52,6 +52,11 @@ public override CompletionInfo Run(ISettingsJobArgs args, CancellationToken ct = /// public override async Task RunAsync(ISettingsJobArgs args, CancellationToken ct) { + _deleted = 0; + _filesmatched = 0; + _filesskipped = 0; + _filesnotfound = 0; + _dircount = 0; _ct = ct; _args = args; int boundedCapacity; diff --git a/HOK.Elastic.Tests/FileSystemCrawlTests.cs b/HOK.Elastic.Tests/FileSystemCrawlTests.cs index a91b631..61f25cd 100644 --- a/HOK.Elastic.Tests/FileSystemCrawlTests.cs +++ b/HOK.Elastic.Tests/FileSystemCrawlTests.cs @@ -1,26 +1,63 @@ +using System.Diagnostics; +using System.IO; +using System.IO.Pipes; +using System.Reflection; +using System.Security.AccessControl; +using System.Security.Cryptography.X509Certificates; +using System.Security.Principal; +using System.Threading.Tasks.Dataflow; +using Elasticsearch.Net.Specification.TextStructureApi; using FluentAssertions; using HOK.Elastic.DAL; using HOK.Elastic.DAL.Models; using HOK.Elastic.FileSystemCrawler; using HOK.Elastic.FileSystemCrawler.Models; using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ClientProtocol; +using Nest; using NLog.Extensions.Logging; +using RtfPipe.Tokens; using Xunit; +using Xunit.Sdk; namespace HOK.Elastic.Tests; public class FileSystemCrawlerFixture : IDisposable { + public string WorkingPathUNC { get; set; } + public CancellationTokenSource CancellationTokenSource { get; set; } public FileSystemCrawlerFixture() { + CancellationTokenSource = new CancellationTokenSource();//v3 we might be able to use TestContext.CancellationTokenSource. // called once before all tests + var machineName = System.Environment.MachineName.ToLowerInvariant(); + string inputPath = @"C:\Elastic\HOK.Elastic Tests\Template\Standard"; string workingPathBase = @"C:\Temp"; string workingPathUnc = @$"\\{System.Environment.MachineName.ToLowerInvariant()}\c$\Temp"; string office = "abc"; + + + tempPath = @$"{workingPathBase}\{office}"; var uri = "https://elastic-dev-cluster:9200"; + if (machineName == "tor-l007") + { + inputPath = @"C:\developer\VS-Projects\HOK.ElasticRoot\crawlertests\Standard"; + workingPathBase = @"C:\developer\VS-Projects\HOK.ElasticRoot"; + workingPathUnc = @$"\\{machineName}\c$\developer\VS-Projects\HOK.ElasticRoot"; + office = "testoffice"; + uri = "https://hok-395vs:9200"; + } + + + var workingPath = $@"{workingPathBase}\{office}\projects\2025\25.12345.00 Crawl Test Project\"; + TestUtils.CopyFilesRecursively(inputPath, workingPath); + workingPath = $@"{workingPathUnc}\{office}\projects\2025\25.12345.00 Crawl Test Project\"; + if (!workingPath.EndsWith("\\")) workingPath = workingPath + "\\"; + var workerargs = new SettingsJobArgs() { // Things to input @@ -41,8 +78,7 @@ public FileSystemCrawlerFixture() $"^{workingPathBase}".Replace("\\", "\\\\") + @"\w{2,4}\\projects\\((\d\d\d\d\\(\d{2}[\d|\.\-]*)+\s*(\+|_|\-)?\s*([^\\|$|\r|\n]*)|(interiors\\|planning\\|architecture\\|hospitality\\)?([^\\]*)?))", PipeCategorizationRegex = - $"^{workingPathBase}".Replace("\\", "\\\\") - + @"(\w{2,4})\\projects\\.*?\\[a-z]\s?\-\s?(?.*?)\\", + @"(?:\\\d{2}\.\d{4,5}\.\d{2}.*?\\[a-z](\-|\s))(?.*?)(?:\\|$)", // Don't Set //PathInclusionRegex = jobSettings.PathInclusionRegex ?? AppSettings.PathInclusionRegex, //FileNameExclusionRegex = jobSettings.FileNameExclusionRegex ?? AppSettings.FileNameExclusionRegex, @@ -55,15 +91,12 @@ public FileSystemCrawlerFixture() var shortDate = date.ToString("yyyyMMdd"); workerargs.IndexNamePrefix = $"testrrunner-{shortDate}."; - var workingPath = - $@"{workingPathBase}\{office}\projects\2025\25.12345.00 Crawl Test Project\"; - TestUtils.CopyFilesRecursively(inputPath, workingPath); - workingPath = - $@"{workingPathUnc}\{office}\projects\2025\25.12345.00 Crawl Test Project\"; + var inputPathBase = new InputPathCollectionBase { new InputPathBase(workingPath, office, PathStatus.Unstarted) }; + WorkingPathUNC = workingPath; inputPathBase.PathForCrawling = workingPathUnc; inputPathBase.PathForCrawlingContent = workingPathBase; inputPathBase.PublishedPath = workingPathBase; @@ -86,7 +119,8 @@ public FileSystemCrawlerFixture() IndexNameHelper indexNameHelper = new IndexNameHelper(workerargs.IndexNamePrefix); PipeLineNameHelper pipeLineNameHelper = new PipeLineNameHelper( workerargs.IndexNamePrefix - ); + ) + { PIPECategorizationProjectExtractRgx = workerargs.PipeCategorizationRegex }; discovery = new DAL.Discovery( pipeLineNameHelper, @@ -121,7 +155,7 @@ public FileSystemCrawlerFixture() PipeLineNameHelper pipeLineHelper = new PipeLineNameHelper(workerargs.IndexNamePrefix); using (var initializationPipeline = new InitializationPipeline(pipeLineNameHelper, indexNameHelper, workerargs.ElasticIndexURI.First(), _loggerFactory.CreateLogger($"{workerargs.JobName}.Setup"))) - { + { if (!initializationPipeline.CheckForPipeLines()) { initializationPipeline.Put(true); @@ -134,8 +168,14 @@ public FileSystemCrawlerFixture() initializationIndex.Put(workerargs.RunningInteractively); } } - } + + TestElasticDAL = new TestElasticDAL( + pipeLineNameHelper, + indexNameHelper, + workerargs.ElasticIndexURI.First(), + _loggerFactory.CreateLogger($"{workerargs.JobName}.Index") + ); } @@ -143,7 +183,7 @@ public void Dispose() { // called once after all tests // TODO: Cleanup tempPath and indices - //File.Delete(tempPath); + //File.Delete(tempPath); } public string tempPath { get; private set; } @@ -155,16 +195,36 @@ public void Dispose() private SecurityHelper securityHelper { get; set; } private DocumentHelper documentHelper { get; set; } public WorkerCrawler worker { get; private set; } + public TestElasticDAL TestElasticDAL { get; set; } } -public class FileSystemCrawlTests : IClassFixture { +public class TestElasticDAL : HOK.Elastic.DAL.Index +{ + public TestElasticDAL(PipeLineNameHelper pipeLineNameHelper, IndexNameHelper indexNameHelper, Uri elastiSearchServerUrl, ILogger logger) : base(pipeLineNameHelper, indexNameHelper, elastiSearchServerUrl, logger) + { + } + public Nest.ElasticClient ElasticClient => this.client; +} + +/// +/// If the test fixture fires multiple times, try to clean and rebuild the solution. +/// + +public class FileSystemCrawlTests : IClassFixture +{ public FileSystemCrawlerFixture fixture; + private ElasticClient ec; + private IndexNameHelper indexHelper; public FileSystemCrawlTests(FileSystemCrawlerFixture fixture) { this.fixture = fixture; + this.ec = this.fixture.TestElasticDAL.ElasticClient; + this.indexHelper = this.fixture.TestElasticDAL.IndexHelper; } + [Fact] + public async void FullCrawlTestSuite() { CancellationTokenSource _ct = new CancellationTokenSource(); @@ -172,15 +232,23 @@ public async void FullCrawlTestSuite() //**Full crawl** fixture.settingsJobArgs.CrawlMode = CrawlMode.Full; CompletionInfo completionInfo = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); - + await FlushIndex(); var path = fixture.worker.GetDirectoriesFromInputPaths(new List { fixture.settingsJobArgs.InputPaths.First() }, fixture.settingsJobArgs).First(); //var path = fixture.settingsJobArgs.InputPaths.First(); if (path == null) + { throw new Exception("Path is not defined"); + } // TODO: This path doesn't match what's in Elastic var crawlContents = fixture.discovery.FindRootAndChildren(path.PublishedPath, true); + if (crawlContents == null) + { + await Task.Delay(30000).ConfigureAwait(false); + crawlContents = fixture.discovery.FindRootAndChildren(path.PublishedPath, true); + } + crawlContents.Should().NotBe(null); //-Check doc count, broken down by index var totalDocCount = crawlContents.Contents.Count(); var dirCount = crawlContents.Contents.Count( @@ -191,7 +259,12 @@ public async void FullCrawlTestSuite() //-Check properties (metadata) //Change permissions + await ChangeAFile(); + await MoveAFile(); //Delete a file + await DeleteAFile(); + await MoveAFolder(); + //**Incremental crawl** //-Check doc count, broken down by index //-Check doc count as different user with less permissions??? @@ -201,6 +274,235 @@ public async void FullCrawlTestSuite() //-Check doc count, broken down by index //-Check doc count as different user with less permissions??? //-Check properties (metadata) + } + private async Task FlushIndex() + { + var response = await ec.Indices.FlushAsync(indexHelper.PrefixWildcard, x => x.Index(indexHelper.AllIndexNames).IgnoreUnavailable(true).Human(true)); + if (response.IsValid) + { + Debug.Write($"Flushed '{response.Shards.Successful}' shards and failed on '{response.Shards.Failures}'"); + } + } + + private async Task DeleteAFile() + { + CancellationTokenSource _ct = fixture.CancellationTokenSource; + long indexCountBefore = 0, indexCountAfter = 0; + + await FlushIndex(); + //**Incremental crawl** + fixture.settingsJobArgs.CrawlMode = CrawlMode.Incremental; + CompletionInfo completionInfo = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + await FlushIndex(); + + //Index Count Before: + var countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + if (countResponse.IsValid) + { + indexCountBefore = countResponse.Count; + } + + //Delete a file (make a change) + FileInfo fi = new FileInfo(Path.Combine(fixture.WorkingPathUNC, "Standard\\F-Specifications\\F7-Other\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant()); + if (fi.Exists) + { + fi.Delete(); + //**Incremental crawl** + CompletionInfo completionInfoAfter = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + var oneDeletedTest = await RetryForSuccess(() => { + countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + if (countResponse.IsValid) + { + indexCountAfter = countResponse.Count; + var indexDiff = indexCountBefore - indexCountAfter; + if (indexDiff == 1) return true; + } + return false; + }); + oneDeletedTest.Should().BeTrue( "Because we deleted one file from the crawl directory"); + var fileCount = (completionInfo.FileCount + completionInfo.FileSkipped) - (completionInfoAfter.FileCount + completionInfoAfter.FileSkipped); + fileCount.Should().Be(1, "Because we deleted one file from the crawl directory"); + } + else + { + false.Should().BeTrue("Source file to delete didn't exist. Can't run test."); + } + } + + private async Task ChangeAFile() + { + CancellationTokenSource _ct = fixture.CancellationTokenSource; + long indexCountBefore = 0, indexCountAfter = 0; + + await FlushIndex(); + //**Incremental crawl** + fixture.settingsJobArgs.CrawlMode = CrawlMode.Incremental; + CompletionInfo completionInfo = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + + await FlushIndex(); + + //Index Count Before: + var countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + if (countResponse.IsValid) + { + indexCountBefore = countResponse.Count; + } + + + //Change a file (make a change) + FileInfo fi = new FileInfo(Path.Combine(fixture.WorkingPathUNC, "Standard\\E-Design\\E6-Models\\_Archive\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant()); + if (fi.Exists) + { + var path = PathHelper.GetPublishedPath(fi.FullName); + //this.fixture.TestElasticDAL. + var fsodocBefore = this.fixture.TestElasticDAL.GetById(path, indexHelper.IndexNameFsoDoc); + + var fileSecurity = fi.GetAccessControl(System.Security.AccessControl.AccessControlSections.Access); + List rulesToAddBack = new List(); + foreach (FileSystemAccessRule rule in fileSecurity.GetAccessRules(true, false, typeof(NTAccount))) + { + rulesToAddBack.Add(rule); + fileSecurity.RemoveAccessRule(rule); + } + fileSecurity.AddAccessRule(new FileSystemAccessRule(new NTAccount("Administrator"), FileSystemRights.Modify, AccessControlType.Allow)); + fi.SetAccessControl(fileSecurity); + //change the date so that it looks like a changed file... + fi.LastWriteTime = fi.CreationTime = DateTime.Now.Subtract(TimeSpan.FromHours(1)); + + //**Incremental crawl** to check for changes + CompletionInfo completionInfoAfter = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + + var aclTest = await RetryForSuccess(() => { + var fsodocAfter = this.fixture.TestElasticDAL.GetById(path, indexHelper.IndexNameFsoDoc); + var aclsAfter = string.Join(",", fsodocAfter.Acls.This); + var aclsBefore = string.Join(",", fsodocBefore.Acls.This); + var differentAcls = aclsAfter != aclsBefore; + return differentAcls; + }); + aclTest.Should().BeTrue(); + var fsodocAfter = this.fixture.TestElasticDAL.GetById(path, indexHelper.IndexNameFsoDoc); + //Index Count After: + countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + if (countResponse.IsValid) + { + indexCountAfter = countResponse.Count; + } + var indexDiff = indexCountBefore - indexCountAfter; + indexDiff.Should().Be(0, "Because we made no change in the total number of files the crawl directory"); + var fileCount = (completionInfo.FileCount + completionInfo.FileSkipped) - (completionInfoAfter.FileCount + completionInfoAfter.FileSkipped); + fileCount.Should().Be(0, "Because we altered one file from the crawl directory"); + completionInfoAfter.FileCount.Should().Be(1, "one new file changed."); + fsodocAfter.Category.Should().Be("DESIGN"); + } + else + { + false.Should().BeTrue(); + } + } + private async Task MoveAFolder() + { + //someday, we can move a folder and incrementalcrawl + //then move a folder and simulate event crawl and ensure the children all get moved (without requiring incremental crawl to discover new files) + true.Should().BeTrue(); + } + + + + private async Task MoveAFile() + { + CancellationTokenSource _ct = fixture.CancellationTokenSource; + long indexCountBefore = 0, indexCountAfter = 0; + + await FlushIndex(); + //**Incremental crawl** + fixture.settingsJobArgs.CrawlMode = CrawlMode.Incremental; + CompletionInfo completionInfo = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + + await FlushIndex(); + + //Index Count Before: + var countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + if (countResponse.IsValid) + { + indexCountBefore = countResponse.Count; + } + + + //Delete a file (make a change) + FileInfo fi = new FileInfo(Path.Combine(fixture.WorkingPathUNC, "Standard\\E-Design\\E6-Models\\_Archive\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant()); + if (fi.Exists) + { + var path = PathHelper.GetPublishedPath(fi.FullName); + //this.fixture.TestElasticDAL. + var fsodocBefore = this.fixture.TestElasticDAL.GetById(path, indexHelper.IndexNameFsoDoc); + var newPath = Path.Combine(fixture.WorkingPathUNC, "Standard\\F-Specifications\\F7-Other\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant(); + if (!File.Exists(newPath)) + { + fi.MoveTo(newPath); + CompletionInfo completionInfoAfter = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + await FlushIndex(); + FSOdocument? fsodocAfter = null; + var foundNewDocument = await RetryForSuccess(() => + { + fsodocAfter = this.fixture.TestElasticDAL.GetById(newPath, indexHelper.IndexNameFsoDoc); + if (fsodocAfter != null) + { + countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + if (countResponse.IsValid) + { + indexCountAfter = countResponse.Count; + if (indexCountAfter == indexCountBefore) + { + return true; + } + } + } + return false; + }); + foundNewDocument.Should().BeTrue(); + fsodocAfter.Should().NotBeNull(); + countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + fsodocAfter?.Category.Should().Be("SPECIFICATIONS"); + var indexDiff = indexCountBefore - indexCountAfter; + indexDiff.Should().Be(0, "Because we made no change in the total number of files the crawl directory"); + completionInfoAfter.FileSkipped.Should().Be(5, "Because we deleted one file when we moved it"); + completionInfoAfter.FileCount.Should().Be(1, "Because we created one file when we moved it"); + completionInfoAfter.Deleted.Should().BeGreaterThan(1, "Because we deleted one file when we moved it"); + } + else + { + false.Should().BeTrue(); + } + } + else + { + false.Should().BeTrue(); + } + } + + /// + /// Method to run a test against the elastic cluster waiting for the search results match the expected outcome. + /// + /// + /// true if successful + public async Task RetryForSuccess(Func f) + { + await FlushIndex(); + DateTime timeout = DateTime.Now.AddMinutes(2); + while (DateTime.Now < timeout)//keep trying until the document and count are updated in search. + { + var result = f.Invoke(); + if (result) + { + return true; + } + else + { + await Task.Delay(10000); + } + } + return false; } } \ No newline at end of file diff --git a/HOK.Elastic.Tests/TestUtils.cs b/HOK.Elastic.Tests/TestUtils.cs index 65855e8..49bfa3f 100644 --- a/HOK.Elastic.Tests/TestUtils.cs +++ b/HOK.Elastic.Tests/TestUtils.cs @@ -1,23 +1,36 @@ +using System.Diagnostics; + namespace HOK.Elastic.Tests; public class TestUtils { public static void CopyFilesRecursively(string sourcePath, string targetPath) { - //Now Create all of the directories - foreach ( - string dirPath in Directory.GetDirectories(sourcePath, "*", SearchOption.AllDirectories) - ) - { - Directory.CreateDirectory(dirPath.Replace(sourcePath, targetPath)); - } + ////Now Create all of the directories + //foreach ( + // string dirPath in Directory.GetDirectories(sourcePath, "*", SearchOption.AllDirectories) + //) + //{ + // Directory.CreateDirectory(dirPath.Replace(sourcePath, targetPath)); + //} - //Copy all the files & Replaces any files with the same name - foreach ( - string newPath in Directory.GetFiles(sourcePath, "*.*", SearchOption.AllDirectories) - ) + ////Copy all the files & Replaces any files with the same name + //foreach ( + // string newPath in Directory.GetFiles(sourcePath, "*.*", SearchOption.AllDirectories) + //) + //{ + // File.Copy(newPath, newPath.Replace(sourcePath, targetPath), true); + //} + ProcessStartInfo processStartInfo = new ProcessStartInfo() { - File.Copy(newPath, newPath.Replace(sourcePath, targetPath), true); - } + UseShellExecute = true, + CreateNoWindow = false, + + FileName = "cmd.exe", + Arguments = $"/k robocopy.exe \"{System.IO.Path.GetDirectoryName(sourcePath)}\" \"{System.IO.Path.GetDirectoryName(targetPath)}\" *.* /e /mir /sec /secfix /r:3 /w:3" + }; + + var p =Process.Start(processStartInfo); + p.WaitForExit(); } } From 0e79487ce3216e5dd9a07143f9eca9c20cc4358b Mon Sep 17 00:00:00 2001 From: jamesblackadar <33873820+jamesblackadar@users.noreply.github.com> Date: Wed, 26 Mar 2025 15:35:43 -0400 Subject: [PATCH 6/9] wip --- HOK.Elastic.Tests/FileSystemCrawlTests.cs | 84 +++++++++++++++++++++-- HOK.Elastic.Tests/TestUtils.cs | 4 +- 2 files changed, 81 insertions(+), 7 deletions(-) diff --git a/HOK.Elastic.Tests/FileSystemCrawlTests.cs b/HOK.Elastic.Tests/FileSystemCrawlTests.cs index 61f25cd..653bb1a 100644 --- a/HOK.Elastic.Tests/FileSystemCrawlTests.cs +++ b/HOK.Elastic.Tests/FileSystemCrawlTests.cs @@ -5,6 +5,7 @@ using System.Security.AccessControl; using System.Security.Cryptography.X509Certificates; using System.Security.Principal; +using System.Text.Json.Serialization; using System.Threading.Tasks.Dataflow; using Elasticsearch.Net.Specification.TextStructureApi; using FluentAssertions; @@ -152,6 +153,14 @@ public FileSystemCrawlerFixture() _loggerFactory.CreateLogger($"{workerargs.JobName}.WorkerCrawler") ); + workerEvents = new WorkerEventStream( + index, + discovery, + securityHelper, + documentHelper, + _loggerFactory.CreateLogger($"{workerargs.JobName}.WorkerEvents") + ); + PipeLineNameHelper pipeLineHelper = new PipeLineNameHelper(workerargs.IndexNamePrefix); using (var initializationPipeline = new InitializationPipeline(pipeLineNameHelper, indexNameHelper, workerargs.ElasticIndexURI.First(), _loggerFactory.CreateLogger($"{workerargs.JobName}.Setup"))) @@ -195,6 +204,7 @@ public void Dispose() private SecurityHelper securityHelper { get; set; } private DocumentHelper documentHelper { get; set; } public WorkerCrawler worker { get; private set; } + public WorkerEventStream workerEvents { get; private set; } public TestElasticDAL TestElasticDAL { get; set; } } @@ -232,6 +242,10 @@ public async void FullCrawlTestSuite() //**Full crawl** fixture.settingsJobArgs.CrawlMode = CrawlMode.Full; CompletionInfo completionInfo = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + + + + await FlushIndex(); var path = fixture.worker.GetDirectoriesFromInputPaths(new List { fixture.settingsJobArgs.InputPaths.First() }, fixture.settingsJobArgs).First(); //var path = fixture.settingsJobArgs.InputPaths.First(); @@ -264,7 +278,8 @@ public async void FullCrawlTestSuite() //Delete a file await DeleteAFile(); await MoveAFolder(); - + + //**Incremental crawl** //-Check doc count, broken down by index //-Check doc count as different user with less permissions??? @@ -304,7 +319,7 @@ private async Task DeleteAFile() } //Delete a file (make a change) - FileInfo fi = new FileInfo(Path.Combine(fixture.WorkingPathUNC, "Standard\\F-Specifications\\F7-Other\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant()); + FileInfo fi = new FileInfo(Path.Combine(fixture.WorkingPathUNC, "F-Specifications\\F7-Other\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant()); if (fi.Exists) { fi.Delete(); @@ -351,7 +366,7 @@ private async Task ChangeAFile() //Change a file (make a change) - FileInfo fi = new FileInfo(Path.Combine(fixture.WorkingPathUNC, "Standard\\E-Design\\E6-Models\\_Archive\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant()); + FileInfo fi = new FileInfo(Path.Combine(fixture.WorkingPathUNC, "E-Design\\E6-Models\\_Archive\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant()); if (fi.Exists) { var path = PathHelper.GetPublishedPath(fi.FullName); @@ -404,7 +419,64 @@ private async Task MoveAFolder() { //someday, we can move a folder and incrementalcrawl //then move a folder and simulate event crawl and ensure the children all get moved (without requiring incremental crawl to discover new files) - true.Should().BeTrue(); + true.Should().BeTrue(); + + CancellationTokenSource _ct = fixture.CancellationTokenSource; + long indexCountBefore = 0, indexCountAfter = 0; + + await FlushIndex(); + //**Incremental crawl** + fixture.settingsJobArgs.CrawlMode = CrawlMode.Incremental; + CompletionInfo completionInfo = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + + await FlushIndex(); + + //Index Count Before: + var countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + if (countResponse.IsValid) + { + indexCountBefore = countResponse.Count; + } + //make a copy + var json = System.Text.Json.JsonSerializer.Serialize(this.fixture.settingsJobArgs); + var settingJobArgCopy = System.Text.Json.JsonSerializer.Deserialize(json); + settingJobArgCopy.InputPaths = new InputPathCollectionEventStream(); + settingJobArgCopy.InputPaths.PathForCrawling = this.fixture.settingsJobArgs.InputPaths.PathForCrawling; + settingJobArgCopy.InputPaths.PathForCrawlingContent = this.fixture.settingsJobArgs.InputPaths.PathForCrawlingContent; + settingJobArgCopy.InputPaths.PublishedPath = this.fixture.settingsJobArgs.InputPaths.PublishedPath; + settingJobArgCopy.CrawlMode = CrawlMode.EventBased; + var newPath = (Path.Combine(fixture.WorkingPathUNC, "G-Engineering2").ToLowerInvariant()); + var oldPath = (Path.Combine(fixture.WorkingPathUNC, "G-Engineering").ToLowerInvariant()); + //var newpath = @"C:\developer\VS-Projects\HOK.ElasticRoot\testoffice\projects\2025\25.12345.00 Crawl Test Project\Standard\G-Engineering2"; + if (Directory.Exists(oldPath)) + { + Directory.Move(oldPath, newPath); + settingJobArgCopy.InputPaths.Add(new InputPathEventStream() { IsDir = true, PathFrom = oldPath, Path = newPath, PresenceAction = ActionPresence.Move, ContentAction = ActionContent.None }); + CompletionInfo completionInfoAfter = await fixture.workerEvents.RunAsync(settingJobArgCopy, _ct.Token); + FSOdirectory? fsoAfter = null; + var contentMoved = await RetryForSuccess(() => + { + fsoAfter = this.fixture.TestElasticDAL.GetById(newPath, indexHelper.IndexNameDir); + if (fsoAfter != null) + { + countResponse = ec.Count(x => x.Index(indexHelper.IndexNameDir)); + if (countResponse.IsValid) + { + indexCountAfter = countResponse.Count; + if (indexCountAfter == indexCountBefore) + { + return true; + } + } + } + return false; + }); + contentMoved.Should().BeTrue(); + } + else + { + false.Should().BeTrue("Source directory to move didn't exist?"); + } } @@ -430,13 +502,13 @@ private async Task MoveAFile() //Delete a file (make a change) - FileInfo fi = new FileInfo(Path.Combine(fixture.WorkingPathUNC, "Standard\\E-Design\\E6-Models\\_Archive\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant()); + FileInfo fi = new FileInfo(Path.Combine(fixture.WorkingPathUNC, "E-Design\\E6-Models\\_Archive\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant()); if (fi.Exists) { var path = PathHelper.GetPublishedPath(fi.FullName); //this.fixture.TestElasticDAL. var fsodocBefore = this.fixture.TestElasticDAL.GetById(path, indexHelper.IndexNameFsoDoc); - var newPath = Path.Combine(fixture.WorkingPathUNC, "Standard\\F-Specifications\\F7-Other\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant(); + var newPath = Path.Combine(fixture.WorkingPathUNC, "F-Specifications\\F7-Other\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant(); if (!File.Exists(newPath)) { fi.MoveTo(newPath); diff --git a/HOK.Elastic.Tests/TestUtils.cs b/HOK.Elastic.Tests/TestUtils.cs index 49bfa3f..f76276a 100644 --- a/HOK.Elastic.Tests/TestUtils.cs +++ b/HOK.Elastic.Tests/TestUtils.cs @@ -21,13 +21,15 @@ public static void CopyFilesRecursively(string sourcePath, string targetPath) //{ // File.Copy(newPath, newPath.Replace(sourcePath, targetPath), true); //} + if (sourcePath.EndsWith("\\")) sourcePath = sourcePath.Substring(0, sourcePath.Length - 1); + if (targetPath.EndsWith("\\")) targetPath = targetPath.Substring(0, targetPath.Length - 1); ProcessStartInfo processStartInfo = new ProcessStartInfo() { UseShellExecute = true, CreateNoWindow = false, FileName = "cmd.exe", - Arguments = $"/k robocopy.exe \"{System.IO.Path.GetDirectoryName(sourcePath)}\" \"{System.IO.Path.GetDirectoryName(targetPath)}\" *.* /e /mir /sec /secfix /r:3 /w:3" + Arguments = $"/k robocopy.exe \"{sourcePath}\" \"{targetPath}\" *.* /e /mir /sec /secfix /r:3 /w:3" }; var p =Process.Start(processStartInfo); From ef6b36a53c5807ad009db481106fcbd9ae2c221c Mon Sep 17 00:00:00 2001 From: jamesblackadar <33873820+jamesblackadar@users.noreply.github.com> Date: Wed, 26 Mar 2025 17:20:46 -0400 Subject: [PATCH 7/9] wip --- HOK.Elastic.Tests/FileSystemCrawlTests.cs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/HOK.Elastic.Tests/FileSystemCrawlTests.cs b/HOK.Elastic.Tests/FileSystemCrawlTests.cs index 653bb1a..be03d56 100644 --- a/HOK.Elastic.Tests/FileSystemCrawlTests.cs +++ b/HOK.Elastic.Tests/FileSystemCrawlTests.cs @@ -456,18 +456,13 @@ private async Task MoveAFolder() FSOdirectory? fsoAfter = null; var contentMoved = await RetryForSuccess(() => { - fsoAfter = this.fixture.TestElasticDAL.GetById(newPath, indexHelper.IndexNameDir); - if (fsoAfter != null) + var crawlContents = fixture.discovery.FindRootAndChildren(newPath, true); + + + if (crawlContents != null) { - countResponse = ec.Count(x => x.Index(indexHelper.IndexNameDir)); - if (countResponse.IsValid) - { - indexCountAfter = countResponse.Count; - if (indexCountAfter == indexCountBefore) - { - return true; - } - } + var y = crawlContents.Contents.Count; + if (y > 1) return true; } return false; }); From 5c72f449dae33b1b57bab43d6e79321322add031 Mon Sep 17 00:00:00 2001 From: jamesblackadar <33873820+jamesblackadar@users.noreply.github.com> Date: Wed, 26 Mar 2025 17:39:50 -0400 Subject: [PATCH 8/9] wip --- HOK.Elastic.Tests/FileSystemCrawlTests.cs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/HOK.Elastic.Tests/FileSystemCrawlTests.cs b/HOK.Elastic.Tests/FileSystemCrawlTests.cs index be03d56..4f6c3a5 100644 --- a/HOK.Elastic.Tests/FileSystemCrawlTests.cs +++ b/HOK.Elastic.Tests/FileSystemCrawlTests.cs @@ -243,7 +243,7 @@ public async void FullCrawlTestSuite() fixture.settingsJobArgs.CrawlMode = CrawlMode.Full; CompletionInfo completionInfo = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); - + await MoveAFolder(); await FlushIndex(); @@ -451,18 +451,26 @@ private async Task MoveAFolder() if (Directory.Exists(oldPath)) { Directory.Move(oldPath, newPath); - settingJobArgCopy.InputPaths.Add(new InputPathEventStream() { IsDir = true, PathFrom = oldPath, Path = newPath, PresenceAction = ActionPresence.Move, ContentAction = ActionContent.None }); + settingJobArgCopy.InputPaths.Add(new InputPathEventStream() { IsDir = true, PathFrom = oldPath, Path = newPath, PresenceAction = ActionPresence.Move, ContentAction = ActionContent.Write }); CompletionInfo completionInfoAfter = await fixture.workerEvents.RunAsync(settingJobArgCopy, _ct.Token); FSOdirectory? fsoAfter = null; var contentMoved = await RetryForSuccess(() => { var crawlContents = fixture.discovery.FindRootAndChildren(newPath, true); - + var descendants = fixture.discovery.FindDescendentsForMoving(newPath); + var descendantCount = descendants.Count(); + var fsodocBefore = this.fixture.TestElasticDAL.GetById(newPath, indexHelper.IndexNameDir); + if(fsodocBefore==null) + { + //this is a bug that we could fix in the future. + //true.Should().BeFalse(); + Debug.Write(newPath + "doc didn't exist but should have"); + } if (crawlContents != null) { - var y = crawlContents.Contents.Count; - if (y > 1) return true; + var children = crawlContents.Contents.Count; + if (children > 1) return true; } return false; }); From 2ec9ffc95b50d37a1266594bcee6ccad3516a83c Mon Sep 17 00:00:00 2001 From: jamesblackadar <33873820+jamesblackadar@users.noreply.github.com> Date: Thu, 27 Mar 2025 11:05:41 -0400 Subject: [PATCH 9/9] wip --- HOK.Elastic.Tests/FileSystemCrawlTests.cs | 32 ++++++++++++----------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/HOK.Elastic.Tests/FileSystemCrawlTests.cs b/HOK.Elastic.Tests/FileSystemCrawlTests.cs index 4f6c3a5..4c37872 100644 --- a/HOK.Elastic.Tests/FileSystemCrawlTests.cs +++ b/HOK.Elastic.Tests/FileSystemCrawlTests.cs @@ -53,7 +53,6 @@ public FileSystemCrawlerFixture() uri = "https://hok-395vs:9200"; } - var workingPath = $@"{workingPathBase}\{office}\projects\2025\25.12345.00 Crawl Test Project\"; TestUtils.CopyFilesRecursively(inputPath, workingPath); workingPath = $@"{workingPathUnc}\{office}\projects\2025\25.12345.00 Crawl Test Project\"; @@ -85,7 +84,6 @@ public FileSystemCrawlerFixture() //FileNameExclusionRegex = jobSettings.FileNameExclusionRegex ?? AppSettings.FileNameExclusionRegex, //CrawlMode = jobSettings.CrawlMode, (will do in the test itself) //InputPathLocation = jobDirectoryInfo.FullName, - ExceptionsPerTenMinuteIntervalLimit = 100 }; DateTime date = DateTime.Now; @@ -242,10 +240,7 @@ public async void FullCrawlTestSuite() //**Full crawl** fixture.settingsJobArgs.CrawlMode = CrawlMode.Full; CompletionInfo completionInfo = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); - await MoveAFolder(); - - await FlushIndex(); var path = fixture.worker.GetDirectoriesFromInputPaths(new List { fixture.settingsJobArgs.InputPaths.First() }, fixture.settingsJobArgs).First(); //var path = fixture.settingsJobArgs.InputPaths.First(); @@ -255,17 +250,20 @@ public async void FullCrawlTestSuite() } // TODO: This path doesn't match what's in Elastic - var crawlContents = fixture.discovery.FindRootAndChildren(path.PublishedPath, true); - if (crawlContents == null) - { - await Task.Delay(30000).ConfigureAwait(false); + DirectoryContents? crawlContents=null; + var oneDeletedTest = await RetryForSuccess(() => { crawlContents = fixture.discovery.FindRootAndChildren(path.PublishedPath, true); - } + if (crawlContents != null) + { + return true; + } + return false; + }); crawlContents.Should().NotBe(null); //-Check doc count, broken down by index - var totalDocCount = crawlContents.Contents.Count(); - var dirCount = crawlContents.Contents.Count( + var totalDocCount = crawlContents?.Contents.Count(); + var dirCount = crawlContents?.Contents.Count( x => x.Item2 == $"{fixture.settingsJobArgs.IndexNamePrefix}dir" ); @@ -297,6 +295,10 @@ private async Task FlushIndex() { Debug.Write($"Flushed '{response.Shards.Successful}' shards and failed on '{response.Shards.Failures}'"); } + else + { + Debug.Write($"Failed to Flush"); + } } @@ -460,13 +462,13 @@ private async Task MoveAFolder() var descendants = fixture.discovery.FindDescendentsForMoving(newPath); var descendantCount = descendants.Count(); var fsodocBefore = this.fixture.TestElasticDAL.GetById(newPath, indexHelper.IndexNameDir); + // if(fsodocBefore==null) { - //this is a bug that we could fix in the future. + //This case is a bug (root directory doesn't seem to get moved...it doesn't cause a problem in search as we are generally only interested in files) //true.Should().BeFalse(); Debug.Write(newPath + "doc didn't exist but should have"); } - if (crawlContents != null) { var children = crawlContents.Contents.Count; @@ -575,7 +577,7 @@ public async Task RetryForSuccess(Func f) } else { - await Task.Delay(10000); + await Task.Delay(10000);//retry again in 10 seconds. } } return false;