diff --git a/HOK.Elastic.ArchiveDiscovery/HOK.Elastic.ArchiveDiscovery.csproj b/HOK.Elastic.ArchiveDiscovery/HOK.Elastic.ArchiveDiscovery.csproj index 5c7388a..4c0e5f8 100644 --- a/HOK.Elastic.ArchiveDiscovery/HOK.Elastic.ArchiveDiscovery.csproj +++ b/HOK.Elastic.ArchiveDiscovery/HOK.Elastic.ArchiveDiscovery.csproj @@ -18,8 +18,8 @@ - - + + diff --git a/HOK.Elastic.DAL/Discovery.cs b/HOK.Elastic.DAL/Discovery.cs index 6a12cbb..71ccad8 100644 --- a/HOK.Elastic.DAL/Discovery.cs +++ b/HOK.Elastic.DAL/Discovery.cs @@ -65,7 +65,6 @@ public DirectoryContents FindRootAndChildren(string directoryPath, bool includeF int pageSize = 1000; int docCount = 0; - var lastCheck = 0; var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = 50 }; bool exit = false; bool firstTimePITUsed = true; @@ -132,6 +131,7 @@ public DirectoryContents FindRootAndChildren(string directoryPath, bool includeF catch (Exception ex) { if (ilerror) _il.LogErr($"Unexpected Error in {nameof(FindRootAndChildren)} because {ex.Message}", "",null,ex ); + exit = true; } } return directoryContents; @@ -614,7 +614,6 @@ public IEnumerable GetIFSOsByQuery(string jsonQueryString, int failureCoun /// public IEnumerable GetIFSOdocumentsLackingContentV2(string directoryPublishedPath, int failureCountFilter, DateTime? minimumDate = null) where T : class, IFSOdocument { - int counter = 0; long docCount = 0; PointInTimeDescriptor pointInTime = null; string pitID = null; diff --git a/HOK.Elastic.DAL/HOK.Elastic.DAL.csproj b/HOK.Elastic.DAL/HOK.Elastic.DAL.csproj index 7987af4..7ac68a6 100644 --- a/HOK.Elastic.DAL/HOK.Elastic.DAL.csproj +++ b/HOK.Elastic.DAL/HOK.Elastic.DAL.csproj @@ -4,12 +4,12 @@ AnyCPU;x64 - - - - + + + + - + diff --git a/HOK.Elastic.DAL/InitializationPipeline.cs b/HOK.Elastic.DAL/InitializationPipeline.cs index 0d10730..27e63a9 100644 --- a/HOK.Elastic.DAL/InitializationPipeline.cs +++ b/HOK.Elastic.DAL/InitializationPipeline.cs @@ -3,6 +3,7 @@ using Nest; using System; using System.Linq; +using System.Text.RegularExpressions; namespace HOK.Elastic.DAL { @@ -174,7 +175,8 @@ private PutPipelineResponse PutPipeValidator() /// example "^\\\\\\\\contoso\\\\projects\\\\.*?\\\\[a-z]\\s?\\-\\s?(?.*?)\\\\", /// private PutPipelineResponse PutPipeCategoryProject() - { + { + PutPipelineResponse response = client.Ingest .PutPipeline(PipeLineNameHelper.PIPECategorizationProject, p => p .Description("Pipeline to assign Category based on filepath.") @@ -182,7 +184,8 @@ private PutPipelineResponse PutPipeCategoryProject() .Grok(g=> g .Field(f=>f.Id) .PatternDefinitions(pd => pd.Add("CATEGORYPATTERN", PipeLineNameHelper.PIPECategorizationProjectExtractRgx)) - .Patterns("%{CATEGORYPATTERN:category}") + //.Patterns("%{CATEGORYPATTERN:category}") + .Patterns("%{CATEGORYPATTERN}") .IgnoreFailure(true) .IgnoreMissing(true) ) diff --git a/HOK.Elastic.FileSystemCrawler.ConsoleProgram/HOK.Elastic.FileSystemCrawler.ConsoleProgram.csproj b/HOK.Elastic.FileSystemCrawler.ConsoleProgram/HOK.Elastic.FileSystemCrawler.ConsoleProgram.csproj index 0ed6d78..1aca741 100644 --- a/HOK.Elastic.FileSystemCrawler.ConsoleProgram/HOK.Elastic.FileSystemCrawler.ConsoleProgram.csproj +++ b/HOK.Elastic.FileSystemCrawler.ConsoleProgram/HOK.Elastic.FileSystemCrawler.ConsoleProgram.csproj @@ -77,26 +77,26 @@ - - + + - - - - - - - - - - - - + + + + + + + + + + + + - + @@ -106,8 +106,8 @@ - - + + diff --git a/HOK.Elastic.FileSystemCrawler.ConsoleProgram/Program.cs b/HOK.Elastic.FileSystemCrawler.ConsoleProgram/Program.cs index 5dd212d..af17920 100644 --- a/HOK.Elastic.FileSystemCrawler.ConsoleProgram/Program.cs +++ b/HOK.Elastic.FileSystemCrawler.ConsoleProgram/Program.cs @@ -20,8 +20,6 @@ using System.Threading.Tasks; using ILogger = Microsoft.Extensions.Logging.ILogger; using LogLevel = Microsoft.Extensions.Logging.LogLevel; -using HOK.Elastic.Logger; -using System.Collections.Generic; namespace HOK.Elastic.FileSystemCrawler.ConsoleProgram { @@ -41,6 +39,7 @@ partial class Program //Main Entry Point static async Task Main(string[] args) { + ///not sure if this is actually needed here or in the msgreader library but it's all working currently... Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); int exitcode = 1; diff --git a/HOK.Elastic.FileSystemCrawler.WebAPI/HOK.Elastic.FileSystemCrawler.WebAPI.csproj b/HOK.Elastic.FileSystemCrawler.WebAPI/HOK.Elastic.FileSystemCrawler.WebAPI.csproj index ad284b6..6dbbfc9 100644 --- a/HOK.Elastic.FileSystemCrawler.WebAPI/HOK.Elastic.FileSystemCrawler.WebAPI.csproj +++ b/HOK.Elastic.FileSystemCrawler.WebAPI/HOK.Elastic.FileSystemCrawler.WebAPI.csproj @@ -11,9 +11,9 @@ - - - + + + diff --git a/HOK.Elastic.FileSystemCrawler/HOK.Elastic.FileSystemCrawler.csproj b/HOK.Elastic.FileSystemCrawler/HOK.Elastic.FileSystemCrawler.csproj index 472e406..fefe79b 100644 --- a/HOK.Elastic.FileSystemCrawler/HOK.Elastic.FileSystemCrawler.csproj +++ b/HOK.Elastic.FileSystemCrawler/HOK.Elastic.FileSystemCrawler.csproj @@ -6,13 +6,13 @@ true - - + + - + - + @@ -23,7 +23,7 @@ - + diff --git a/HOK.Elastic.FileSystemCrawler/Models/InputPathCollectionEventStream.cs b/HOK.Elastic.FileSystemCrawler/Models/InputPathCollectionEventStream.cs index abec45f..1de7983 100644 --- a/HOK.Elastic.FileSystemCrawler/Models/InputPathCollectionEventStream.cs +++ b/HOK.Elastic.FileSystemCrawler/Models/InputPathCollectionEventStream.cs @@ -31,7 +31,7 @@ public void Add(InputPathEventStream[] newItems) } } } - public new void Add(InputPathEventStream newItem) + public void Add(InputPathEventStream newItem) { if (newItem == null) return; diff --git a/HOK.Elastic.FileSystemCrawler/WorkerBase.cs b/HOK.Elastic.FileSystemCrawler/WorkerBase.cs index 76e45b5..1b2c38b 100644 --- a/HOK.Elastic.FileSystemCrawler/WorkerBase.cs +++ b/HOK.Elastic.FileSystemCrawler/WorkerBase.cs @@ -66,7 +66,7 @@ public WorkerBase(IIndex elasticIngest, IDiscovery elasticDiscovery, DocumentHel public abstract CompletionInfo Run(ISettingsJobArgs args, CancellationToken ct = default); public abstract Task RunAsync(ISettingsJobArgs args, CancellationToken ct = default); - internal List GetDirectoriesFromInputPaths(List setofPaths, ISettingsJobArgs args)///TODO why not just use args.inputpaths? + public List GetDirectoriesFromInputPaths(List setofPaths, ISettingsJobArgs args)///TODO why not just use args.inputpaths? { var inputs = new List(); foreach (var item in setofPaths) diff --git a/HOK.Elastic.FileSystemCrawler/WorkerCrawler.cs b/HOK.Elastic.FileSystemCrawler/WorkerCrawler.cs index 6b9031b..b05f982 100644 --- a/HOK.Elastic.FileSystemCrawler/WorkerCrawler.cs +++ b/HOK.Elastic.FileSystemCrawler/WorkerCrawler.cs @@ -52,6 +52,11 @@ public override CompletionInfo Run(ISettingsJobArgs args, CancellationToken ct = /// public override async Task RunAsync(ISettingsJobArgs args, CancellationToken ct) { + _deleted = 0; + _filesmatched = 0; + _filesskipped = 0; + _filesnotfound = 0; + _dircount = 0; _ct = ct; _args = args; int boundedCapacity; diff --git a/HOK.Elastic.Logger/HOK.Elastic.Logger.csproj b/HOK.Elastic.Logger/HOK.Elastic.Logger.csproj index f02d67e..37ca6cb 100644 --- a/HOK.Elastic.Logger/HOK.Elastic.Logger.csproj +++ b/HOK.Elastic.Logger/HOK.Elastic.Logger.csproj @@ -4,8 +4,8 @@ AnyCPU;x64 - - + + diff --git a/HOK.Elastic.RoleMappingGroupSync/HOK.Elastic.RoleMappingGroupSync.csproj b/HOK.Elastic.RoleMappingGroupSync/HOK.Elastic.RoleMappingGroupSync.csproj index 4f2ce52..17b5dc7 100644 --- a/HOK.Elastic.RoleMappingGroupSync/HOK.Elastic.RoleMappingGroupSync.csproj +++ b/HOK.Elastic.RoleMappingGroupSync/HOK.Elastic.RoleMappingGroupSync.csproj @@ -5,8 +5,8 @@ AnyCPU;x64 - - + + diff --git a/HOK.Elastic.Tests/FileSystemCrawlTests.cs b/HOK.Elastic.Tests/FileSystemCrawlTests.cs new file mode 100644 index 0000000..4c37872 --- /dev/null +++ b/HOK.Elastic.Tests/FileSystemCrawlTests.cs @@ -0,0 +1,585 @@ +using System.Diagnostics; +using System.IO; +using System.IO.Pipes; +using System.Reflection; +using System.Security.AccessControl; +using System.Security.Cryptography.X509Certificates; +using System.Security.Principal; +using System.Text.Json.Serialization; +using System.Threading.Tasks.Dataflow; +using Elasticsearch.Net.Specification.TextStructureApi; +using FluentAssertions; +using HOK.Elastic.DAL; +using HOK.Elastic.DAL.Models; +using HOK.Elastic.FileSystemCrawler; +using HOK.Elastic.FileSystemCrawler.Models; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ClientProtocol; +using Nest; +using NLog.Extensions.Logging; +using RtfPipe.Tokens; +using Xunit; +using Xunit.Sdk; + +namespace HOK.Elastic.Tests; + +public class FileSystemCrawlerFixture : IDisposable +{ + public string WorkingPathUNC { get; set; } + public CancellationTokenSource CancellationTokenSource { get; set; } + public FileSystemCrawlerFixture() + { + CancellationTokenSource = new CancellationTokenSource();//v3 we might be able to use TestContext.CancellationTokenSource. + // called once before all tests + var machineName = System.Environment.MachineName.ToLowerInvariant(); + + string inputPath = @"C:\Elastic\HOK.Elastic Tests\Template\Standard"; + string workingPathBase = @"C:\Temp"; + string workingPathUnc = @$"\\{System.Environment.MachineName.ToLowerInvariant()}\c$\Temp"; + string office = "abc"; + + + + tempPath = @$"{workingPathBase}\{office}"; + + var uri = "https://elastic-dev-cluster:9200"; + if (machineName == "tor-l007") + { + inputPath = @"C:\developer\VS-Projects\HOK.ElasticRoot\crawlertests\Standard"; + workingPathBase = @"C:\developer\VS-Projects\HOK.ElasticRoot"; + workingPathUnc = @$"\\{machineName}\c$\developer\VS-Projects\HOK.ElasticRoot"; + office = "testoffice"; + uri = "https://hok-395vs:9200"; + } + + var workingPath = $@"{workingPathBase}\{office}\projects\2025\25.12345.00 Crawl Test Project\"; + TestUtils.CopyFilesRecursively(inputPath, workingPath); + workingPath = $@"{workingPathUnc}\{office}\projects\2025\25.12345.00 Crawl Test Project\"; + if (!workingPath.EndsWith("\\")) workingPath = workingPath + "\\"; + + var workerargs = new SettingsJobArgs() + { + // Things to input + ElasticDiscoveryURI = new List { new Uri(uri) }, + ElasticIndexURI = new List { new Uri(uri) }, + //ReadFileContents = jobSettings.ReadFileContents, + //PathForCrawling = jobSettings.PathForCrawling, + //PathForCrawlingContent = jobSettings.PathForCrawlingContent, + //PublishedPath = jobSettings.PublishedPath, + + DocInsertionThreads = 1, + CPUCoreThreadMultiplier = 1, + ReadContentSizeLimitMB = 200, + BulkUploadSize = 500, + IgnoreExtensions = new List(), + OfficeSiteExtractRegex = $"^{workingPathBase}".Replace("\\", "\\\\") + @"(\w{2,4})", + ProjectExtractRegex = + $"^{workingPathBase}".Replace("\\", "\\\\") + + @"\w{2,4}\\projects\\((\d\d\d\d\\(\d{2}[\d|\.\-]*)+\s*(\+|_|\-)?\s*([^\\|$|\r|\n]*)|(interiors\\|planning\\|architecture\\|hospitality\\)?([^\\]*)?))", + PipeCategorizationRegex = + @"(?:\\\d{2}\.\d{4,5}\.\d{2}.*?\\[a-z](\-|\s))(?.*?)(?:\\|$)", + // Don't Set + //PathInclusionRegex = jobSettings.PathInclusionRegex ?? AppSettings.PathInclusionRegex, + //FileNameExclusionRegex = jobSettings.FileNameExclusionRegex ?? AppSettings.FileNameExclusionRegex, + //CrawlMode = jobSettings.CrawlMode, (will do in the test itself) + //InputPathLocation = jobDirectoryInfo.FullName, + ExceptionsPerTenMinuteIntervalLimit = 100 + }; + DateTime date = DateTime.Now; + var shortDate = date.ToString("yyyyMMdd"); + workerargs.IndexNamePrefix = $"testrrunner-{shortDate}."; + + + var inputPathBase = new InputPathCollectionBase + { + new InputPathBase(workingPath, office, PathStatus.Unstarted) + }; + WorkingPathUNC = workingPath; + inputPathBase.PathForCrawling = workingPathUnc; + inputPathBase.PathForCrawlingContent = workingPathBase; + inputPathBase.PublishedPath = workingPathBase; + workerargs.InputPaths = inputPathBase; + workerargs.PathForCrawling = workingPathUnc; + workerargs.PathForCrawlingContent = workingPathUnc; + workerargs.PublishedPath = workingPathUnc; + workerargs.InputPathLocation = workingPathBase; + workerargs.RunningInteractively = false; + PathHelper.Set(workerargs.PublishedPath, workerargs.PathForCrawlingContent, workerargs.PathForCrawling); + + PathHelper.SetPathInclusion(workerargs.PathInclusionRegex); + PathHelper.SetFileNameExclusion(workerargs.FileNameExclusionRegex); + PathHelper.SetOfficeExtractRgx(workerargs.OfficeSiteExtractRegex); + PathHelper.SetProjectExtractRgx(workerargs.ProjectExtractRegex); + PathHelper.IgnoreExtensions = workerargs.IgnoreExtensions?.Distinct().ToHashSet(); + //fileSystemCrawlTest = new FileSystemCrawlTest(workerargs); + settingsJobArgs = workerargs; + _loggerFactory = LoggerFactory.Create(builder => builder.AddNLog()); + IndexNameHelper indexNameHelper = new IndexNameHelper(workerargs.IndexNamePrefix); + PipeLineNameHelper pipeLineNameHelper = new PipeLineNameHelper( + workerargs.IndexNamePrefix + ) + { PIPECategorizationProjectExtractRgx = workerargs.PipeCategorizationRegex }; + + discovery = new DAL.Discovery( + pipeLineNameHelper, + indexNameHelper, + workerargs.ElasticDiscoveryURI.First(), + _loggerFactory.CreateLogger($"{workerargs.JobName}.Discovery") + ); + index = new DAL.Index( + pipeLineNameHelper, + indexNameHelper, + workerargs.ElasticIndexURI.First(), + _loggerFactory.CreateLogger($"{workerargs.JobName}.Index") + ); + securityHelper = new SecurityHelper( + _loggerFactory.CreateLogger($"{workerargs.JobName}.SecurityHelper") + ); + documentHelper = new DocumentHelper( + workerargs.ReadFileContents ?? false, + securityHelper, + index, + _loggerFactory.CreateLogger($"{workerargs.JobName}.DocumentHelper") + ); + + worker = new WorkerCrawler( + index, + discovery, + securityHelper, + documentHelper, + _loggerFactory.CreateLogger($"{workerargs.JobName}.WorkerCrawler") + ); + + workerEvents = new WorkerEventStream( + index, + discovery, + securityHelper, + documentHelper, + _loggerFactory.CreateLogger($"{workerargs.JobName}.WorkerEvents") + ); + + PipeLineNameHelper pipeLineHelper = new PipeLineNameHelper(workerargs.IndexNamePrefix); + + using (var initializationPipeline = new InitializationPipeline(pipeLineNameHelper, indexNameHelper, workerargs.ElasticIndexURI.First(), _loggerFactory.CreateLogger($"{workerargs.JobName}.Setup"))) + { + if (!initializationPipeline.CheckForPipeLines()) + { + initializationPipeline.Put(true); + } + + using (var initializationIndex = new InitializationIndex(pipeLineNameHelper, indexNameHelper, workerargs.ElasticIndexURI.First(), _loggerFactory.CreateLogger($"{workerargs.JobName}.IndexSetup"))) + { + if (initializationIndex.PreFlightFail()) + { + initializationIndex.Put(workerargs.RunningInteractively); + } + } + } + + TestElasticDAL = new TestElasticDAL( + pipeLineNameHelper, + indexNameHelper, + workerargs.ElasticIndexURI.First(), + _loggerFactory.CreateLogger($"{workerargs.JobName}.Index") + ); + } + + + public void Dispose() + { + // called once after all tests + // TODO: Cleanup tempPath and indices + //File.Delete(tempPath); + } + + public string tempPath { get; private set; } + public SettingsJobArgs settingsJobArgs { get; private set; } + public CompletionInfo? completionInfo; + private readonly ILoggerFactory _loggerFactory; + public DAL.Discovery discovery { get; private set; } + private DAL.Index index { get; set; } + private SecurityHelper securityHelper { get; set; } + private DocumentHelper documentHelper { get; set; } + public WorkerCrawler worker { get; private set; } + public WorkerEventStream workerEvents { get; private set; } + public TestElasticDAL TestElasticDAL { get; set; } +} + +public class TestElasticDAL : HOK.Elastic.DAL.Index +{ + public TestElasticDAL(PipeLineNameHelper pipeLineNameHelper, IndexNameHelper indexNameHelper, Uri elastiSearchServerUrl, ILogger logger) : base(pipeLineNameHelper, indexNameHelper, elastiSearchServerUrl, logger) + { + } + public Nest.ElasticClient ElasticClient => this.client; +} + +/// +/// If the test fixture fires multiple times, try to clean and rebuild the solution. +/// + +public class FileSystemCrawlTests : IClassFixture +{ + public FileSystemCrawlerFixture fixture; + private ElasticClient ec; + private IndexNameHelper indexHelper; + public FileSystemCrawlTests(FileSystemCrawlerFixture fixture) + { + this.fixture = fixture; + this.ec = this.fixture.TestElasticDAL.ElasticClient; + this.indexHelper = this.fixture.TestElasticDAL.IndexHelper; + } + + + [Fact] + + public async void FullCrawlTestSuite() + { + CancellationTokenSource _ct = new CancellationTokenSource(); + + //**Full crawl** + fixture.settingsJobArgs.CrawlMode = CrawlMode.Full; + CompletionInfo completionInfo = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + await MoveAFolder(); + await FlushIndex(); + var path = fixture.worker.GetDirectoriesFromInputPaths(new List { fixture.settingsJobArgs.InputPaths.First() }, fixture.settingsJobArgs).First(); + //var path = fixture.settingsJobArgs.InputPaths.First(); + if (path == null) + { + throw new Exception("Path is not defined"); + } + + // TODO: This path doesn't match what's in Elastic + DirectoryContents? crawlContents=null; + var oneDeletedTest = await RetryForSuccess(() => { + crawlContents = fixture.discovery.FindRootAndChildren(path.PublishedPath, true); + if (crawlContents != null) + { + return true; + } + return false; + }); + + crawlContents.Should().NotBe(null); + //-Check doc count, broken down by index + var totalDocCount = crawlContents?.Contents.Count(); + var dirCount = crawlContents?.Contents.Count( + x => x.Item2 == $"{fixture.settingsJobArgs.IndexNamePrefix}dir" + ); + + dirCount.Should().BeGreaterOrEqualTo(1); + + //-Check properties (metadata) + //Change permissions + await ChangeAFile(); + await MoveAFile(); + //Delete a file + await DeleteAFile(); + await MoveAFolder(); + + + //**Incremental crawl** + //-Check doc count, broken down by index + //-Check doc count as different user with less permissions??? + //-Check properties (metadata) + //Add a file + //**Incremental crawl** + //-Check doc count, broken down by index + //-Check doc count as different user with less permissions??? + //-Check properties (metadata) + } + private async Task FlushIndex() + { + var response = await ec.Indices.FlushAsync(indexHelper.PrefixWildcard, x => x.Index(indexHelper.AllIndexNames).IgnoreUnavailable(true).Human(true)); + if (response.IsValid) + { + Debug.Write($"Flushed '{response.Shards.Successful}' shards and failed on '{response.Shards.Failures}'"); + } + else + { + Debug.Write($"Failed to Flush"); + } + } + + + private async Task DeleteAFile() + { + CancellationTokenSource _ct = fixture.CancellationTokenSource; + long indexCountBefore = 0, indexCountAfter = 0; + + await FlushIndex(); + //**Incremental crawl** + fixture.settingsJobArgs.CrawlMode = CrawlMode.Incremental; + CompletionInfo completionInfo = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + await FlushIndex(); + + //Index Count Before: + var countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + if (countResponse.IsValid) + { + indexCountBefore = countResponse.Count; + } + + //Delete a file (make a change) + FileInfo fi = new FileInfo(Path.Combine(fixture.WorkingPathUNC, "F-Specifications\\F7-Other\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant()); + if (fi.Exists) + { + fi.Delete(); + //**Incremental crawl** + CompletionInfo completionInfoAfter = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + var oneDeletedTest = await RetryForSuccess(() => { + countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + if (countResponse.IsValid) + { + indexCountAfter = countResponse.Count; + var indexDiff = indexCountBefore - indexCountAfter; + if (indexDiff == 1) return true; + } + return false; + }); + oneDeletedTest.Should().BeTrue( "Because we deleted one file from the crawl directory"); + var fileCount = (completionInfo.FileCount + completionInfo.FileSkipped) - (completionInfoAfter.FileCount + completionInfoAfter.FileSkipped); + fileCount.Should().Be(1, "Because we deleted one file from the crawl directory"); + } + else + { + false.Should().BeTrue("Source file to delete didn't exist. Can't run test."); + } + } + + private async Task ChangeAFile() + { + CancellationTokenSource _ct = fixture.CancellationTokenSource; + long indexCountBefore = 0, indexCountAfter = 0; + + await FlushIndex(); + //**Incremental crawl** + fixture.settingsJobArgs.CrawlMode = CrawlMode.Incremental; + CompletionInfo completionInfo = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + + await FlushIndex(); + + //Index Count Before: + var countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + if (countResponse.IsValid) + { + indexCountBefore = countResponse.Count; + } + + + //Change a file (make a change) + FileInfo fi = new FileInfo(Path.Combine(fixture.WorkingPathUNC, "E-Design\\E6-Models\\_Archive\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant()); + if (fi.Exists) + { + var path = PathHelper.GetPublishedPath(fi.FullName); + //this.fixture.TestElasticDAL. + var fsodocBefore = this.fixture.TestElasticDAL.GetById(path, indexHelper.IndexNameFsoDoc); + + var fileSecurity = fi.GetAccessControl(System.Security.AccessControl.AccessControlSections.Access); + List rulesToAddBack = new List(); + foreach (FileSystemAccessRule rule in fileSecurity.GetAccessRules(true, false, typeof(NTAccount))) + { + rulesToAddBack.Add(rule); + fileSecurity.RemoveAccessRule(rule); + } + fileSecurity.AddAccessRule(new FileSystemAccessRule(new NTAccount("Administrator"), FileSystemRights.Modify, AccessControlType.Allow)); + fi.SetAccessControl(fileSecurity); + //change the date so that it looks like a changed file... + fi.LastWriteTime = fi.CreationTime = DateTime.Now.Subtract(TimeSpan.FromHours(1)); + + //**Incremental crawl** to check for changes + CompletionInfo completionInfoAfter = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + + var aclTest = await RetryForSuccess(() => { + var fsodocAfter = this.fixture.TestElasticDAL.GetById(path, indexHelper.IndexNameFsoDoc); + var aclsAfter = string.Join(",", fsodocAfter.Acls.This); + var aclsBefore = string.Join(",", fsodocBefore.Acls.This); + var differentAcls = aclsAfter != aclsBefore; + return differentAcls; + }); + aclTest.Should().BeTrue(); + var fsodocAfter = this.fixture.TestElasticDAL.GetById(path, indexHelper.IndexNameFsoDoc); + //Index Count After: + countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + if (countResponse.IsValid) + { + indexCountAfter = countResponse.Count; + } + var indexDiff = indexCountBefore - indexCountAfter; + indexDiff.Should().Be(0, "Because we made no change in the total number of files the crawl directory"); + var fileCount = (completionInfo.FileCount + completionInfo.FileSkipped) - (completionInfoAfter.FileCount + completionInfoAfter.FileSkipped); + fileCount.Should().Be(0, "Because we altered one file from the crawl directory"); + completionInfoAfter.FileCount.Should().Be(1, "one new file changed."); + fsodocAfter.Category.Should().Be("DESIGN"); + } + else + { + false.Should().BeTrue(); + } + } + private async Task MoveAFolder() + { + //someday, we can move a folder and incrementalcrawl + //then move a folder and simulate event crawl and ensure the children all get moved (without requiring incremental crawl to discover new files) + true.Should().BeTrue(); + + CancellationTokenSource _ct = fixture.CancellationTokenSource; + long indexCountBefore = 0, indexCountAfter = 0; + + await FlushIndex(); + //**Incremental crawl** + fixture.settingsJobArgs.CrawlMode = CrawlMode.Incremental; + CompletionInfo completionInfo = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + + await FlushIndex(); + + //Index Count Before: + var countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + if (countResponse.IsValid) + { + indexCountBefore = countResponse.Count; + } + //make a copy + var json = System.Text.Json.JsonSerializer.Serialize(this.fixture.settingsJobArgs); + var settingJobArgCopy = System.Text.Json.JsonSerializer.Deserialize(json); + settingJobArgCopy.InputPaths = new InputPathCollectionEventStream(); + settingJobArgCopy.InputPaths.PathForCrawling = this.fixture.settingsJobArgs.InputPaths.PathForCrawling; + settingJobArgCopy.InputPaths.PathForCrawlingContent = this.fixture.settingsJobArgs.InputPaths.PathForCrawlingContent; + settingJobArgCopy.InputPaths.PublishedPath = this.fixture.settingsJobArgs.InputPaths.PublishedPath; + settingJobArgCopy.CrawlMode = CrawlMode.EventBased; + var newPath = (Path.Combine(fixture.WorkingPathUNC, "G-Engineering2").ToLowerInvariant()); + var oldPath = (Path.Combine(fixture.WorkingPathUNC, "G-Engineering").ToLowerInvariant()); + //var newpath = @"C:\developer\VS-Projects\HOK.ElasticRoot\testoffice\projects\2025\25.12345.00 Crawl Test Project\Standard\G-Engineering2"; + if (Directory.Exists(oldPath)) + { + Directory.Move(oldPath, newPath); + settingJobArgCopy.InputPaths.Add(new InputPathEventStream() { IsDir = true, PathFrom = oldPath, Path = newPath, PresenceAction = ActionPresence.Move, ContentAction = ActionContent.Write }); + CompletionInfo completionInfoAfter = await fixture.workerEvents.RunAsync(settingJobArgCopy, _ct.Token); + FSOdirectory? fsoAfter = null; + var contentMoved = await RetryForSuccess(() => + { + var crawlContents = fixture.discovery.FindRootAndChildren(newPath, true); + var descendants = fixture.discovery.FindDescendentsForMoving(newPath); + var descendantCount = descendants.Count(); + var fsodocBefore = this.fixture.TestElasticDAL.GetById(newPath, indexHelper.IndexNameDir); + // + if(fsodocBefore==null) + { + //This case is a bug (root directory doesn't seem to get moved...it doesn't cause a problem in search as we are generally only interested in files) + //true.Should().BeFalse(); + Debug.Write(newPath + "doc didn't exist but should have"); + } + if (crawlContents != null) + { + var children = crawlContents.Contents.Count; + if (children > 1) return true; + } + return false; + }); + contentMoved.Should().BeTrue(); + } + else + { + false.Should().BeTrue("Source directory to move didn't exist?"); + } + } + + + + private async Task MoveAFile() + { + CancellationTokenSource _ct = fixture.CancellationTokenSource; + long indexCountBefore = 0, indexCountAfter = 0; + + await FlushIndex(); + //**Incremental crawl** + fixture.settingsJobArgs.CrawlMode = CrawlMode.Incremental; + CompletionInfo completionInfo = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + + await FlushIndex(); + + //Index Count Before: + var countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + if (countResponse.IsValid) + { + indexCountBefore = countResponse.Count; + } + + + //Delete a file (make a change) + FileInfo fi = new FileInfo(Path.Combine(fixture.WorkingPathUNC, "E-Design\\E6-Models\\_Archive\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant()); + if (fi.Exists) + { + var path = PathHelper.GetPublishedPath(fi.FullName); + //this.fixture.TestElasticDAL. + var fsodocBefore = this.fixture.TestElasticDAL.GetById(path, indexHelper.IndexNameFsoDoc); + var newPath = Path.Combine(fixture.WorkingPathUNC, "F-Specifications\\F7-Other\\!ARCHIVE IN ZIP FILES ONLY.txt").ToLowerInvariant(); + if (!File.Exists(newPath)) + { + fi.MoveTo(newPath); + CompletionInfo completionInfoAfter = await fixture.worker.RunAsync(fixture.settingsJobArgs, _ct.Token); + await FlushIndex(); + FSOdocument? fsodocAfter = null; + var foundNewDocument = await RetryForSuccess(() => + { + fsodocAfter = this.fixture.TestElasticDAL.GetById(newPath, indexHelper.IndexNameFsoDoc); + if (fsodocAfter != null) + { + countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + if (countResponse.IsValid) + { + indexCountAfter = countResponse.Count; + if (indexCountAfter == indexCountBefore) + { + return true; + } + } + } + return false; + }); + foundNewDocument.Should().BeTrue(); + fsodocAfter.Should().NotBeNull(); + countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + countResponse = ec.Count(x => x.Index(indexHelper.IndexNameFsoDoc)); + fsodocAfter?.Category.Should().Be("SPECIFICATIONS"); + var indexDiff = indexCountBefore - indexCountAfter; + indexDiff.Should().Be(0, "Because we made no change in the total number of files the crawl directory"); + completionInfoAfter.FileSkipped.Should().Be(5, "Because we deleted one file when we moved it"); + completionInfoAfter.FileCount.Should().Be(1, "Because we created one file when we moved it"); + completionInfoAfter.Deleted.Should().BeGreaterThan(1, "Because we deleted one file when we moved it"); + } + else + { + false.Should().BeTrue(); + } + } + else + { + false.Should().BeTrue(); + } + } + + /// + /// Method to run a test against the elastic cluster waiting for the search results match the expected outcome. + /// + /// + /// true if successful + public async Task RetryForSuccess(Func f) + { + await FlushIndex(); + DateTime timeout = DateTime.Now.AddMinutes(2); + while (DateTime.Now < timeout)//keep trying until the document and count are updated in search. + { + var result = f.Invoke(); + if (result) + { + return true; + } + else + { + await Task.Delay(10000);//retry again in 10 seconds. + } + } + return false; + } +} \ No newline at end of file diff --git a/HOK.Elastic.Tests/HOK.Elastic.Tests.csproj b/HOK.Elastic.Tests/HOK.Elastic.Tests.csproj new file mode 100644 index 0000000..62514bd --- /dev/null +++ b/HOK.Elastic.Tests/HOK.Elastic.Tests.csproj @@ -0,0 +1,22 @@ + + + net8.0-windows + enable + enable + + + + + + + + + + + + + + + + + diff --git a/HOK.Elastic.Tests/HOK.Elastic.Tests.sln b/HOK.Elastic.Tests/HOK.Elastic.Tests.sln new file mode 100644 index 0000000..7837958 --- /dev/null +++ b/HOK.Elastic.Tests/HOK.Elastic.Tests.sln @@ -0,0 +1,24 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.2.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HOK.Elastic.Tests", "HOK.Elastic.Tests.csproj", "{9D1A765D-0600-5F8C-4676-855FB237CE1F}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {9D1A765D-0600-5F8C-4676-855FB237CE1F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9D1A765D-0600-5F8C-4676-855FB237CE1F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9D1A765D-0600-5F8C-4676-855FB237CE1F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9D1A765D-0600-5F8C-4676-855FB237CE1F}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {FD065F66-79D3-4308-9C3E-E63672F67936} + EndGlobalSection +EndGlobal diff --git a/HOK.Elastic.Tests/LocalCluster.cs b/HOK.Elastic.Tests/LocalCluster.cs new file mode 100644 index 0000000..436a97f --- /dev/null +++ b/HOK.Elastic.Tests/LocalCluster.cs @@ -0,0 +1,22 @@ +using Elastic.Elasticsearch.Xunit; +using Elastic.Elasticsearch.Ephemeral; +using Elastic.Elasticsearch.Ephemeral.Plugins; +using Elastic.Stack.ArtifactsApi.Products; + +namespace HOK.Elastic.Tests +{ + /// Declare our cluster that we want to inject into our test classes + public class LocalCluster : XunitClusterBase + { + /// + /// We pass our configuration instance to the base class. + /// + public LocalCluster() : base(new XunitClusterConfiguration( + "8.17.2", + ClusterFeatures.None, + new ElasticsearchPlugins(new List {}), + 1 + )) + { } + } +} diff --git a/HOK.Elastic.Tests/TestUtils.cs b/HOK.Elastic.Tests/TestUtils.cs new file mode 100644 index 0000000..f76276a --- /dev/null +++ b/HOK.Elastic.Tests/TestUtils.cs @@ -0,0 +1,38 @@ +using System.Diagnostics; + +namespace HOK.Elastic.Tests; + +public class TestUtils +{ + public static void CopyFilesRecursively(string sourcePath, string targetPath) + { + ////Now Create all of the directories + //foreach ( + // string dirPath in Directory.GetDirectories(sourcePath, "*", SearchOption.AllDirectories) + //) + //{ + // Directory.CreateDirectory(dirPath.Replace(sourcePath, targetPath)); + //} + + ////Copy all the files & Replaces any files with the same name + //foreach ( + // string newPath in Directory.GetFiles(sourcePath, "*.*", SearchOption.AllDirectories) + //) + //{ + // File.Copy(newPath, newPath.Replace(sourcePath, targetPath), true); + //} + if (sourcePath.EndsWith("\\")) sourcePath = sourcePath.Substring(0, sourcePath.Length - 1); + if (targetPath.EndsWith("\\")) targetPath = targetPath.Substring(0, targetPath.Length - 1); + ProcessStartInfo processStartInfo = new ProcessStartInfo() + { + UseShellExecute = true, + CreateNoWindow = false, + + FileName = "cmd.exe", + Arguments = $"/k robocopy.exe \"{sourcePath}\" \"{targetPath}\" *.* /e /mir /sec /secfix /r:3 /w:3" + }; + + var p =Process.Start(processStartInfo); + p.WaitForExit(); + } +} diff --git a/HOK.Elastic.sln b/HOK.Elastic.sln index f510b95..f0c4156 100644 --- a/HOK.Elastic.sln +++ b/HOK.Elastic.sln @@ -18,6 +18,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HOK.Elastic.FileSystemCrawl EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HOK.Elastic.ArchiveDiscovery", "HOK.Elastic.ArchiveDiscovery\HOK.Elastic.ArchiveDiscovery.csproj", "{7633149C-6D28-4CAE-B931-CD1080092ACD}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HOK.Elastic.Tests", "HOK.Elastic.Tests\HOK.Elastic.Tests.csproj", "{EF90D6C1-F8EC-45FE-A7C7-3527865F413F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -90,6 +92,14 @@ Global {7633149C-6D28-4CAE-B931-CD1080092ACD}.Release|Any CPU.Build.0 = Release|Any CPU {7633149C-6D28-4CAE-B931-CD1080092ACD}.Release|x64.ActiveCfg = Release|Any CPU {7633149C-6D28-4CAE-B931-CD1080092ACD}.Release|x64.Build.0 = Release|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Debug|x64.ActiveCfg = Debug|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Debug|x64.Build.0 = Debug|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Release|Any CPU.Build.0 = Release|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Release|x64.ActiveCfg = Release|Any CPU + {EF90D6C1-F8EC-45FE-A7C7-3527865F413F}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/HOK.NasuniAuditEventAPI/HOK.NasuniAuditEventAPI.csproj b/HOK.NasuniAuditEventAPI/HOK.NasuniAuditEventAPI.csproj index a2048f6..e0d1c0e 100644 --- a/HOK.NasuniAuditEventAPI/HOK.NasuniAuditEventAPI.csproj +++ b/HOK.NasuniAuditEventAPI/HOK.NasuniAuditEventAPI.csproj @@ -15,9 +15,9 @@ - - - + + +