Skip to content

FileSystemCrawler Tests #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
djsiroky wants to merge 9 commits into
base: dev
Choose a base branch
from
Draft

FileSystemCrawler Tests #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
3f133f1
Add tests project
djsiroky Mar 17, 2025
951bc23
Update references based on vulnerability warnings
djsiroky Mar 17, 2025
23c48a8
Changes to existing library based on adding test project
djsiroky Mar 17, 2025
558c947
Rename Tests.cs to FileSystemCrawlTests.cs
djsiroky Mar 17, 2025
c7b3adc
wip
jamesblackadar Mar 26, 2025
0e79487
wip
jamesblackadar Mar 26, 2025
ef6b36a
wip
jamesblackadar Mar 26, 2025
5c72f44
wip
jamesblackadar Mar 26, 2025
2ec9ffc
wip
jamesblackadar Mar 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions HOK.Elastic.ArchiveDiscovery/HOK.Elastic.ArchiveDiscovery.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
<None Update="nlog.config" CopyToOutputDirectory="Always" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.2" />
<PackageReference Include="NLog" Version="5.3.2" />
<PackageReference Include="NLog.Extensions.Logging" Version="5.3.11" />
</ItemGroup>
Expand Down
3 changes: 1 addition & 2 deletions HOK.Elastic.DAL/Discovery.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ public DirectoryContents FindRootAndChildren(string directoryPath, bool includeF

int pageSize = 1000;
int docCount = 0;
var lastCheck = 0;
var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = 50 };
bool exit = false;
bool firstTimePITUsed = true;
Expand Down Expand Up @@ -132,6 +131,7 @@ public DirectoryContents FindRootAndChildren(string directoryPath, bool includeF
catch (Exception ex)
{
if (ilerror) _il.LogErr($"Unexpected Error in {nameof(FindRootAndChildren)} because {ex.Message}", "",null,ex );
exit = true;
}
}
return directoryContents;
Expand Down Expand Up @@ -614,7 +614,6 @@ public IEnumerable<T> GetIFSOsByQuery<T>(string jsonQueryString, int failureCoun
/// <returns></returns>
public IEnumerable<T> GetIFSOdocumentsLackingContentV2<T>(string directoryPublishedPath, int failureCountFilter, DateTime? minimumDate = null) where T : class, IFSOdocument
{
int counter = 0;
long docCount = 0;
PointInTimeDescriptor pointInTime = null;
string pitID = null;
Expand Down
10 changes: 5 additions & 5 deletions HOK.Elastic.DAL/HOK.Elastic.DAL.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@
<Platforms>AnyCPU;x64</Platforms>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Elasticsearch.Net" Version="7.17.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
<PackageReference Include="NEST" Version="7.17.0" />
<PackageReference Include="Elasticsearch.Net" Version="7.17.5" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.2" />
<PackageReference Include="NEST" Version="7.17.5" />
<PackageReference Include="System.Buffers" Version="4.5.1" />
<PackageReference Include="System.Diagnostics.DiagnosticSource" Version="6.0.0" />
<PackageReference Include="System.Diagnostics.DiagnosticSource" Version="9.0.2" />
<PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
<PackageReference Include="System.IO.FileSystem.AccessControl" Version="5.0.0" />
<PackageReference Include="System.Net.NameResolution" Version="4.3.0" />
Expand Down
7 changes: 5 additions & 2 deletions HOK.Elastic.DAL/InitializationPipeline.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using Nest;
using System;
using System.Linq;
using System.Text.RegularExpressions;

namespace HOK.Elastic.DAL
{
Expand Down Expand Up @@ -174,15 +175,17 @@ private PutPipelineResponse PutPipeValidator()
/// example "^\\\\\\\\contoso\\\\projects\\\\.*?\\\\[a-z]\\s?\\-\\s?(?<category>.*?)\\\\",
/// </summary>
private PutPipelineResponse PutPipeCategoryProject()
{
{

PutPipelineResponse response = client.Ingest
.PutPipeline(PipeLineNameHelper.PIPECategorizationProject, p => p
.Description("Pipeline to assign Category based on filepath.")
.Processors(pr => pr
.Grok<FSO>(g=> g
.Field(f=>f.Id)
.PatternDefinitions(pd => pd.Add("CATEGORYPATTERN", PipeLineNameHelper.PIPECategorizationProjectExtractRgx))
.Patterns("%{CATEGORYPATTERN:category}")
//.Patterns("%{CATEGORYPATTERN:category}")
.Patterns("%{CATEGORYPATTERN}")
.IgnoreFailure(true)
.IgnoreMissing(true)
)
Expand Down
34 changes: 17 additions & 17 deletions ...stic.FileSystemCrawler.ConsoleProgram/HOK.Elastic.FileSystemCrawler.ConsoleProgram.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,26 +77,26 @@
</None>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Elasticsearch.Net" Version="7.17.0" />
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
<PackageReference Include="Elasticsearch.Net" Version="7.17.5" />
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.2" />
<PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
<PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.2" />
<PackageReference Include="Microsoft.Extensions.Primitives" Version="8.0.0" />
<PackageReference Include="NEST" Version="7.17.0" />
<PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Options" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Primitives" Version="9.0.2" />
<PackageReference Include="NEST" Version="7.17.5" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Newtonsoft.Json.Schema" Version="3.0.14" />
<PackageReference Include="NLog" Version="5.3.2" />
<PackageReference Include="NLog.Extensions.Logging" Version="5.3.11" />
<PackageReference Include="RestSharp" Version="110.2.0" />
<PackageReference Include="RestSharp" Version="112.1.0" />
<PackageReference Include="System.Buffers" Version="4.5.1" />
<PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
<PackageReference Include="System.Data.DataSetExtensions" Version="4.5.0" />
Expand All @@ -106,8 +106,8 @@
<PackageReference Include="System.Security.AccessControl" Version="6.0.0" />
<PackageReference Include="System.Security.Permissions" Version="8.0.0" />
<PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />
<PackageReference Include="System.Text.Encodings.Web" Version="8.0.0" />
<PackageReference Include="System.Text.Json" Version="8.0.3" />
<PackageReference Include="System.Text.Encodings.Web" Version="9.0.2" />
<PackageReference Include="System.Text.Json" Version="9.0.2" />
<PackageReference Include="System.Threading.Tasks.Dataflow" Version="8.0.0" />
</ItemGroup>
<ProjectExtensions>
Expand Down
3 changes: 1 addition & 2 deletions HOK.Elastic.FileSystemCrawler.ConsoleProgram/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@
using System.Threading.Tasks;
using ILogger = Microsoft.Extensions.Logging.ILogger;
using LogLevel = Microsoft.Extensions.Logging.LogLevel;
using HOK.Elastic.Logger;
using System.Collections.Generic;

namespace HOK.Elastic.FileSystemCrawler.ConsoleProgram
{
Expand All @@ -41,6 +39,7 @@ partial class Program
//Main Entry Point
static async Task<int> Main(string[] args)
{

///not sure if this is actually needed here or in the msgreader library but it's all working currently...
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
int exitcode = 1;
Expand Down
6 changes: 3 additions & 3 deletions HOK.Elastic.FileSystemCrawler.WebAPI/HOK.Elastic.FileSystemCrawler.WebAPI.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@
<None Remove="log4net\**" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.4" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="8.0.4" />
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="8.0.2" />
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="9.0.2" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="9.0.2" />
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="9.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="NLog" Version="5.3.2" />
<PackageReference Include="NLog.Extensions.Logging" Version="5.3.11" />
Expand Down
10 changes: 5 additions & 5 deletions HOK.Elastic.FileSystemCrawler/HOK.Elastic.FileSystemCrawler.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@
<RunAnalyzersDuringBuild>true</RunAnalyzersDuringBuild>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Elasticsearch.Net" Version="7.17.0" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
<PackageReference Include="Elasticsearch.Net" Version="7.17.5" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.2" />
<PackageReference Include="MsgReader" Version="5.2.0" />
<PackageReference Include="NEST" Version="7.17.0" />
<PackageReference Include="NEST" Version="7.17.5" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="OpenMcdf" Version="2.3.0" />
<PackageReference Include="RestSharp" Version="110.2.0" />
<PackageReference Include="RestSharp" Version="112.1.0" />
<PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
<PackageReference Include="System.Configuration.ConfigurationManager" Version="8.0.0" />
<PackageReference Include="System.DirectoryServices.AccountManagement" Version="5.0.0" />
Expand All @@ -23,7 +23,7 @@
<PackageReference Include="System.Net.Sockets" Version="4.3.0" />
<PackageReference Include="System.Runtime.Caching" Version="5.0.0" />
<PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />
<PackageReference Include="System.Text.Json" Version="8.0.3" />
<PackageReference Include="System.Text.Json" Version="9.0.2" />
<PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.11.1" />
</ItemGroup>
<ItemGroup>
Expand Down
2 changes: 1 addition & 1 deletion HOK.Elastic.FileSystemCrawler/Models/InputPathCollectionEventStream.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public void Add(InputPathEventStream[] newItems)
}
}
}
public new void Add(InputPathEventStream newItem)
public void Add(InputPathEventStream newItem)
{
if (newItem == null) return;

Expand Down
2 changes: 1 addition & 1 deletion HOK.Elastic.FileSystemCrawler/WorkerBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public WorkerBase(IIndex elasticIngest, IDiscovery elasticDiscovery, DocumentHel
public abstract CompletionInfo Run(ISettingsJobArgs args, CancellationToken ct = default);
public abstract Task<CompletionInfo> RunAsync(ISettingsJobArgs args, CancellationToken ct = default);

internal List<FSOdirectory> GetDirectoriesFromInputPaths(List<InputPathBase> setofPaths, ISettingsJobArgs args)///TODO why not just use args.inputpaths?
public List<FSOdirectory> GetDirectoriesFromInputPaths(List<InputPathBase> setofPaths, ISettingsJobArgs args)///TODO why not just use args.inputpaths?
{
var inputs = new List<FSOdirectory>();
foreach (var item in setofPaths)
Expand Down
5 changes: 5 additions & 0 deletions HOK.Elastic.FileSystemCrawler/WorkerCrawler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ public override CompletionInfo Run(ISettingsJobArgs args, CancellationToken ct =
/// <returns></returns>
public override async Task<CompletionInfo> RunAsync(ISettingsJobArgs args, CancellationToken ct)
{
_deleted = 0;
_filesmatched = 0;
_filesskipped = 0;
_filesnotfound = 0;
_dircount = 0;
_ct = ct;
_args = args;
int boundedCapacity;
Expand Down
4 changes: 2 additions & 2 deletions HOK.Elastic.Logger/HOK.Elastic.Logger.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
<Platforms>AnyCPU;x64</Platforms>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.2" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="System.Configuration.ConfigurationManager" Version="8.0.0" />
</ItemGroup>
Expand Down
4 changes: 2 additions & 2 deletions HOK.Elastic.RoleMappingGroupSync/HOK.Elastic.RoleMappingGroupSync.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
<Platforms>AnyCPU;x64</Platforms>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Elasticsearch.Net" Version="7.17.0" />
<PackageReference Include="NEST" Version="7.17.0" />
<PackageReference Include="Elasticsearch.Net" Version="7.17.5" />
<PackageReference Include="NEST" Version="7.17.5" />
<PackageReference Include="NLog" Version="5.3.2" />
<PackageReference Include="NLog.Extensions.Logging" Version="5.3.11" />
<PackageReference Include="System.Configuration.ConfigurationManager" Version="8.0.0" />
Expand Down
Loading