From 275abe8ebda666414873ac201bb671c128d55323 Mon Sep 17 00:00:00 2001 From: Meik Date: Tue, 16 Jun 2026 14:47:21 +0200 Subject: [PATCH] Add NTFS data area diagnostics logging --- LiamNtfs/C4IT.LIAM.Ntfs.cs | 239 ++++++++++++++++++++++++++++++++++++- LiamNtfs/cNtfsBase.cs | 48 +++++++- 2 files changed, 283 insertions(+), 4 deletions(-) diff --git a/LiamNtfs/C4IT.LIAM.Ntfs.cs b/LiamNtfs/C4IT.LIAM.Ntfs.cs index 814d859..6b6a410 100644 --- a/LiamNtfs/C4IT.LIAM.Ntfs.cs +++ b/LiamNtfs/C4IT.LIAM.Ntfs.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.DirectoryServices.AccountManagement; using System.IO; using System.Linq; @@ -52,6 +53,72 @@ namespace C4IT.LIAM public int Level { get; set; } = -1; } + private sealed class cNtfsDataAreaDiagnostics + { + private const int MaxSamples = 10; + + public int ScanRequests { get; set; } + public int FoldersEnumerated { get; set; } + public int EnumerationFailures { get; set; } + public int MetadataFailures { get; set; } + public int SubtreeFailures { get; set; } + public int DataAreaCandidates { get; set; } + public int IncludedDataAreas { get; set; } + public int SkippedByDataAreaRegex { get; set; } + public int SkippedByExcludeRule { get; set; } + public int SkippedByMissingIncludeRule { get; set; } + public int SkippedSubtreesByExcludeRule { get; set; } + public int SkippedSubtreesByMissingIncludeRule { get; set; } + public int AclReadFailures { get; set; } + public int AclEntriesEvaluated { get; set; } + public int UnresolvedAclSids { get; set; } + public int AclGroupsWithoutNamingMatch { get; set; } + public int DataAreasWithoutPermissionMapping { get; set; } + public List EnumerationFailureSamples { get; } = new List(); + public List FilterSamples { get; } = new List(); + public List MappingIssueSamples { get; } = new List(); + + public void MergeEnumeration(cNtfsEnumerationDiagnostics diagnostics) + { + if (diagnostics == null) + return; + + ScanRequests++; + FoldersEnumerated += diagnostics.FoldersEnumerated; + EnumerationFailures += diagnostics.EnumerationFailures; + MetadataFailures += diagnostics.MetadataFailures; + SubtreeFailures += diagnostics.SubtreeFailures; + AddSamples(EnumerationFailureSamples, diagnostics.FailureSamples); + } + + public void AddFilterSample(string sample) + { + AddSample(FilterSamples, sample); + } + + public void AddMappingIssueSample(string sample) + { + AddSample(MappingIssueSamples, sample); + } + + private static void AddSamples(List target, IEnumerable samples) + { + if (target == null || samples == null) + return; + + foreach (var sample in samples) + AddSample(target, sample); + } + + private static void AddSample(List target, string sample) + { + if (target == null || string.IsNullOrWhiteSpace(sample) || target.Count >= MaxSamples) + return; + + target.Add(sample); + } + } + public static Guid nftsModuleId = new Guid("77e213a1-6517-ea11-4881-000c2980fd94"); private const string AdditionalConfigurationExcludePathsKey = "NtfsExcludePaths"; private const string AdditionalConfigurationIncludePathsKey = "NtfsIncludePaths"; @@ -63,6 +130,7 @@ namespace C4IT.LIAM public readonly cActiveDirectoryBase activeDirectoryBase = new cActiveDirectoryBase(); private readonly Dictionary> publishedShareCache = new Dictionary>(StringComparer.OrdinalIgnoreCase); private readonly Dictionary dfsEntryPathCache = new Dictionary(StringComparer.OrdinalIgnoreCase); + private cNtfsDataAreaDiagnostics currentDataAreaDiagnostics; //public readonly bool WithoutPrivateFolders = true; @@ -163,6 +231,8 @@ namespace C4IT.LIAM { var CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); + var stopwatch = Stopwatch.StartNew(); + currentDataAreaDiagnostics = new cNtfsDataAreaDiagnostics(); try { if (!cC4ITLicenseM42ESM.Instance.IsValid || !cC4ITLicenseM42ESM.Instance.Modules.ContainsKey(nftsModuleId)) @@ -171,22 +241,37 @@ namespace C4IT.LIAM return new List(); } if (!await LogonAsync()) + { + LogEntry($"NTFS getDataAreas failed. Stage=Logon RootPath='{RootPath}', Error='{GetLastErrorMessage()}'", LogLevels.Warning); return null; + } if (string.IsNullOrEmpty(this.RootPath)) + { + LogEntry("NTFS getDataAreas failed. Stage=Configuration RootPath is empty.", LogLevels.Warning); return null; + } + LogNtfsAdContext(); var DataAreas = new List(); var rootClassification = ClassifyPath(this.RootPath); var rootDataArea = await BuildDataAreaAsync(rootClassification); if (rootDataArea == null) + { + LogEntry($"NTFS getDataAreas failed. Stage=RootClassification RootPath='{RootPath}', PathKind='{rootClassification?.Kind}', NormalizedPath='{rootClassification?.NormalizedPath}'", LogLevels.Warning); return null; + } DataAreas.Add(rootDataArea); + currentDataAreaDiagnostics.IncludedDataAreas++; if (Depth == 0) + { + LogDataAreaScanSummary(DataAreas, stopwatch.Elapsed); return DataAreas; + } DataAreas.AddRange(await GetChildDataAreasAsync(rootClassification, Depth)); + LogDataAreaScanSummary(DataAreas, stopwatch.Elapsed); return DataAreas; } catch (Exception E) @@ -399,8 +484,12 @@ namespace C4IT.LIAM } var folderEntries = await ntfsBase.RequestFoldersListAsync(parentClassification.NormalizedPath, 1); + currentDataAreaDiagnostics?.MergeEnumeration(ntfsBase.LastEnumerationDiagnostics); if (folderEntries == null) + { + LogEntry($"NTFS scan returned null while enumerating children of '{parentClassification.NormalizedPath}'. ChildrenSkipped=true.", LogLevels.Warning); return children; + } foreach (var entry in folderEntries.Values.OfType()) { @@ -439,23 +528,47 @@ namespace C4IT.LIAM if (classification == null) return false; + if (currentDataAreaDiagnostics != null) + currentDataAreaDiagnostics.DataAreaCandidates++; + if (!MatchesDataAreaRegEx(classification.DisplayName)) + { + if (currentDataAreaDiagnostics != null) + { + currentDataAreaDiagnostics.SkippedByDataAreaRegex++; + currentDataAreaDiagnostics.AddFilterSample($"DataAreaRegEx rejected '{classification.NormalizedPath}' with DisplayName='{classification.DisplayName}' and DataAreaRegEx='{this.DataAreaRegEx}'"); + } + LogEntry($"Skip NTFS path '{classification.NormalizedPath}' because DisplayName='{classification.DisplayName}' does not match DataAreaRegEx='{this.DataAreaRegEx}'", LogLevels.Debug); return false; + } string matchingConfigurationKey; string matchingRule; if (IsPathBlacklisted(classification, out matchingConfigurationKey, out matchingRule)) { + if (currentDataAreaDiagnostics != null) + { + currentDataAreaDiagnostics.SkippedByExcludeRule++; + currentDataAreaDiagnostics.AddFilterSample($"Exclude rejected '{classification.NormalizedPath}' via {matchingConfigurationKey}={matchingRule}"); + } LogEntry($"Skip NTFS path '{classification.NormalizedPath}' due to AdditionalConfiguration rule '{matchingConfigurationKey}={matchingRule}'", LogLevels.Debug); return false; } if (!IsPathWhitelisted(classification, true, out matchingConfigurationKey, out matchingRule)) { + if (currentDataAreaDiagnostics != null) + { + currentDataAreaDiagnostics.SkippedByMissingIncludeRule++; + currentDataAreaDiagnostics.AddFilterSample($"Include missing for '{classification.NormalizedPath}' via {AdditionalConfigurationIncludePathsKey}"); + } LogEntry($"Skip NTFS path '{classification.NormalizedPath}' because no AdditionalConfiguration whitelist matched", LogLevels.Debug); return false; } + if (currentDataAreaDiagnostics != null) + currentDataAreaDiagnostics.IncludedDataAreas++; + return true; } @@ -468,6 +581,11 @@ namespace C4IT.LIAM string matchingRule; if (IsPathBlacklisted(classification, out matchingConfigurationKey, out matchingRule)) { + if (currentDataAreaDiagnostics != null) + { + currentDataAreaDiagnostics.SkippedSubtreesByExcludeRule++; + currentDataAreaDiagnostics.AddFilterSample($"Subtree exclude rejected '{classification.NormalizedPath}' via {matchingConfigurationKey}={matchingRule}"); + } LogEntry($"Skip NTFS subtree '{classification.NormalizedPath}' due to AdditionalConfiguration rule '{matchingConfigurationKey}={matchingRule}'", LogLevels.Debug); return false; } @@ -478,6 +596,11 @@ namespace C4IT.LIAM if (IsPathWhitelisted(classification, true, out matchingConfigurationKey, out matchingRule)) return true; + if (currentDataAreaDiagnostics != null) + { + currentDataAreaDiagnostics.SkippedSubtreesByMissingIncludeRule++; + currentDataAreaDiagnostics.AddFilterSample($"Subtree outside include whitelist '{classification.NormalizedPath}' via {AdditionalConfigurationIncludePathsKey}"); + } LogEntry($"Skip NTFS subtree '{classification.NormalizedPath}' because it is outside AdditionalConfiguration whitelist '{AdditionalConfigurationIncludePathsKey}'", LogLevels.Debug); return false; } @@ -681,6 +804,99 @@ namespace C4IT.LIAM return Regex.IsMatch(valueSegment, regexPattern, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); } + private void LogNtfsAdContext() + { + var includePaths = string.Join(";", GetAdditionalConfigurationValues(AdditionalConfigurationIncludePathsKey)); + var excludePaths = string.Join(";", GetAdditionalConfigurationValues(AdditionalConfigurationExcludePathsKey)); + var dc = activeDirectoryBase?.EffectiveDomainController; + + LogEntry( + $"NTFS getDataAreas context: Stage=Start RootPath='{RootPath}', MaxDepth={MaxDepth}, Domain='{Domain}', DomainController='{(string.IsNullOrWhiteSpace(dc) ? "(domain locator)" : dc)}', Credential='{Credential?.Identification}', GroupStrategy='{GroupStrategy}', DataAreaRegEx='{DataAreaRegEx}', IncludePaths='{includePaths}', ExcludePaths='{excludePaths}'", + LogLevels.Info); + } + + private void LogDataAreaScanSummary(ICollection dataAreas, TimeSpan duration) + { + var diagnostics = currentDataAreaDiagnostics; + if (diagnostics == null) + return; + + var returnedDataAreas = dataAreas?.Count ?? 0; + var permissionDataAreas = dataAreas?.OfType().Count() ?? 0; + var issueCount = + diagnostics.EnumerationFailures + + diagnostics.MetadataFailures + + diagnostics.SubtreeFailures + + diagnostics.AclReadFailures + + diagnostics.UnresolvedAclSids + + diagnostics.AclGroupsWithoutNamingMatch + + diagnostics.DataAreasWithoutPermissionMapping; + + LogEntry( + $"NTFS getDataAreas summary: Stage=Summary RootPath='{RootPath}', MaxDepth={MaxDepth}, ReturnedDataAreas={returnedDataAreas}, PermissionDataAreas={permissionDataAreas}, ScanRequests={diagnostics.ScanRequests}, FoldersEnumerated={diagnostics.FoldersEnumerated}, DataAreaCandidates={diagnostics.DataAreaCandidates}, IncludedCandidates={diagnostics.IncludedDataAreas}, SkippedByDataAreaRegEx={diagnostics.SkippedByDataAreaRegex}, SkippedByExcludeRule={diagnostics.SkippedByExcludeRule}, SkippedByMissingIncludeRule={diagnostics.SkippedByMissingIncludeRule}, SkippedSubtreesByExcludeRule={diagnostics.SkippedSubtreesByExcludeRule}, SkippedSubtreesByMissingIncludeRule={diagnostics.SkippedSubtreesByMissingIncludeRule}, EnumerationFailures={diagnostics.EnumerationFailures}, MetadataFailures={diagnostics.MetadataFailures}, SubtreeFailures={diagnostics.SubtreeFailures}, AclReadFailures={diagnostics.AclReadFailures}, AclEntriesEvaluated={diagnostics.AclEntriesEvaluated}, UnresolvedAclSids={diagnostics.UnresolvedAclSids}, AclGroupsWithoutNamingMatch={diagnostics.AclGroupsWithoutNamingMatch}, DataAreasWithoutPermissionMapping={diagnostics.DataAreasWithoutPermissionMapping}, Duration='{duration}'", + issueCount > 0 ? LogLevels.Warning : LogLevels.Info); + + if (diagnostics.EnumerationFailureSamples.Count > 0) + LogEntry($"NTFS diagnostic samples: Stage=Enumeration Samples='{string.Join(" | ", diagnostics.EnumerationFailureSamples)}'", LogLevels.Warning); + + if (diagnostics.FilterSamples.Count > 0) + LogEntry($"NTFS diagnostic samples: Stage=Filter Samples='{string.Join(" | ", diagnostics.FilterSamples)}'", LogLevels.Info); + + if (diagnostics.MappingIssueSamples.Count > 0) + LogEntry($"NTFS diagnostic samples: Stage=AclMapping Samples='{string.Join(" | ", diagnostics.MappingIssueSamples)}'", LogLevels.Warning); + } + + internal void RecordAclReadFailure(string path) + { + if (currentDataAreaDiagnostics != null) + { + currentDataAreaDiagnostics.AclReadFailures++; + currentDataAreaDiagnostics.AddMappingIssueSample($"ACL read failed for '{path}'"); + } + + LogEntry($"NTFS data area ACL read failed. Stage=AclRead Path='{path}'", LogLevels.Warning); + } + + internal void RecordAclEntryEvaluated() + { + if (currentDataAreaDiagnostics != null) + currentDataAreaDiagnostics.AclEntriesEvaluated++; + } + + internal void RecordUnresolvedAclSid(string path, string sid) + { + if (currentDataAreaDiagnostics != null) + { + currentDataAreaDiagnostics.UnresolvedAclSids++; + currentDataAreaDiagnostics.AddMappingIssueSample($"Unresolved ACL SID '{sid}' on '{path}'"); + } + + LogEntry($"NTFS ACL SID could not be resolved to an AD group. Stage=AclSidResolution Path='{path}', Sid='{sid}'", LogLevels.Warning); + } + + internal void RecordAclGroupWithoutNamingMatch(string path, string samAccountName) + { + if (currentDataAreaDiagnostics != null) + { + currentDataAreaDiagnostics.AclGroupsWithoutNamingMatch++; + currentDataAreaDiagnostics.AddMappingIssueSample($"ACL group '{samAccountName}' on '{path}' did not match naming conventions"); + } + } + + internal void RecordDataAreaWithoutPermissionMapping(string path, IEnumerable aclGroups, string ownerWildcard, string writeWildcard, string readWildcard) + { + var groups = string.Join(",", aclGroups ?? Enumerable.Empty()); + if (currentDataAreaDiagnostics != null) + { + currentDataAreaDiagnostics.DataAreasWithoutPermissionMapping++; + currentDataAreaDiagnostics.AddMappingIssueSample($"DataArea '{path}' has no Owner/Write/Read mapping. ACL groups: {groups}"); + } + + LogEntry( + $"NTFS data area returned without Owner/Write/Read group mapping. Stage=AclMapping Path='{path}', AclGroups='{groups}', OwnerWildcard='{ownerWildcard}', WriteWildcard='{writeWildcard}', ReadWildcard='{readWildcard}'", + LogLevels.Warning); + } + private List GetDfsObjectPrefixes(string path) { var normalizedPath = NormalizeUncPath(path); @@ -1347,22 +1563,30 @@ namespace C4IT.LIAM { var ACLs = Provider.activeDirectoryBase.GetAccessControlList(path); if (ACLs == null) + { + Provider.RecordAclReadFailure(path); return; + } var ownerNamingConvention = Provider.NamingConventions.First(i => i.AccessRole == eLiamAccessRoles.Owner && (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGP && i.Scope == eLiamAccessRoleScopes.Global || Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP && i.Scope == eLiamAccessRoleScopes.DomainLocal)); var writeNamingConvention = Provider.NamingConventions.First(i => i.AccessRole == eLiamAccessRoles.Write && (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGP && i.Scope == eLiamAccessRoleScopes.Global || Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP && i.Scope == eLiamAccessRoleScopes.DomainLocal)); var readNamingConvention = Provider.NamingConventions.First(i => i.AccessRole == eLiamAccessRoles.Read && (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGP && i.Scope == eLiamAccessRoleScopes.Global || Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP && i.Scope == eLiamAccessRoleScopes.DomainLocal)); var traverseNamingConvention = Provider.NamingConventions.First(i => i.AccessRole == eLiamAccessRoles.Traverse); + var resolvedAclGroups = new List(); + var matchedOwner = false; + var matchedWrite = false; + var matchedRead = false; foreach (FileSystemAccessRule rule in ACLs) { var aclSid = rule.IdentityReference.Value; if (aclSid == "S-1-1-0") continue; + Provider.RecordAclEntryEvaluated(); GroupPrincipal grp = GroupPrincipal.FindByIdentity(Provider.activeDirectoryBase.adContext, IdentityType.Sid, aclSid); if (grp == null) { - DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' could not be resolved to an AD group. Naming convention matching skipped."); + Provider.RecordUnresolvedAclSid(path, aclSid); continue; } @@ -1373,8 +1597,10 @@ namespace C4IT.LIAM continue; } + resolvedAclGroups.Add(samAccountName); if (Regex.IsMatch(samAccountName, ownerNamingConvention.Wildcard, RegexOptions.IgnoreCase)) { + matchedOwner = true; this.OwnerGroupIdentifier = aclSid; DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' resolved to '{samAccountName}' and matched Owner naming convention '{ownerNamingConvention.Wildcard}'."); if (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP) @@ -1401,6 +1627,7 @@ namespace C4IT.LIAM } else if (Regex.IsMatch(samAccountName, writeNamingConvention.Wildcard, RegexOptions.IgnoreCase)) { + matchedWrite = true; this.WriteGroupIdentifier = aclSid; DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' resolved to '{samAccountName}' and matched Write naming convention '{writeNamingConvention.Wildcard}'."); if (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP) @@ -1427,6 +1654,7 @@ namespace C4IT.LIAM } else if (Regex.IsMatch(samAccountName, readNamingConvention.Wildcard, RegexOptions.IgnoreCase)) { + matchedRead = true; this.ReadGroupIdentifier = aclSid; DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' resolved to '{samAccountName}' and matched Read naming convention '{readNamingConvention.Wildcard}'."); if (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP) @@ -1458,9 +1686,18 @@ namespace C4IT.LIAM } else { + Provider.RecordAclGroupWithoutNamingMatch(path, samAccountName); DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' resolved to '{samAccountName}', but did not match Owner/Write/Read/Traverse naming conventions."); } } + + if (!matchedOwner && !matchedWrite && !matchedRead) + Provider.RecordDataAreaWithoutPermissionMapping( + path, + resolvedAclGroups, + ownerNamingConvention.Wildcard, + writeNamingConvention.Wildcard, + readNamingConvention.Wildcard); } } diff --git a/LiamNtfs/cNtfsBase.cs b/LiamNtfs/cNtfsBase.cs index 7124cbb..e879fe8 100644 --- a/LiamNtfs/cNtfsBase.cs +++ b/LiamNtfs/cNtfsBase.cs @@ -25,6 +25,7 @@ namespace LiamNtfs public PrincipalContext adContext = null; public Exception LastException { get; private set; } = null; public string LastErrorMessage { get; private set; } = null; + public cNtfsEnumerationDiagnostics LastEnumerationDiagnostics { get; private set; } = new cNtfsEnumerationDiagnostics(); [MethodImpl(MethodImplOptions.AggressiveInlining)] public void ResetError() @@ -169,6 +170,7 @@ namespace LiamNtfs public async Task RequestFoldersListAsync(string rootPath, int depth) { ResetError(); + LastEnumerationDiagnostics = new cNtfsEnumerationDiagnostics(rootPath, depth); try { await Task.Delay(0); @@ -217,7 +219,9 @@ namespace LiamNtfs } catch (Exception E) { - cLogManager.LogEntry($"Could not enumerate directories under '{rootPath.FullName}': {E.Message}", LogLevels.Warning); + LastEnumerationDiagnostics.EnumerationFailures++; + LastEnumerationDiagnostics.AddFailure(rootPath.FullName, E); + cLogManager.LogEntry($"NTFS enumeration failed under '{rootPath.FullName}'. ReturnedPartialResult=true. Error='{E.Message}'", LogLevels.Warning); cLogManager.LogException(E, LogLevels.Debug); return folders; } @@ -239,11 +243,14 @@ namespace LiamNtfs } catch (Exception E) { - cLogManager.LogEntry($"Could not read directory metadata for '{directory.FullName}': {E.Message}", LogLevels.Warning); + LastEnumerationDiagnostics.MetadataFailures++; + LastEnumerationDiagnostics.AddFailure(directory.FullName, E); + cLogManager.LogEntry($"NTFS directory metadata read failed for '{directory.FullName}'. DirectorySkipped=true. Error='{E.Message}'", LogLevels.Warning); cLogManager.LogException(E, LogLevels.Debug); continue; } + LastEnumerationDiagnostics.FoldersEnumerated++; folders.Add(folder); if (depth <= 0) continue; @@ -256,7 +263,9 @@ namespace LiamNtfs } catch (Exception E) { - cLogManager.LogEntry($"Could not scan subtree '{directory.FullName}': {E.Message}", LogLevels.Warning); + LastEnumerationDiagnostics.SubtreeFailures++; + LastEnumerationDiagnostics.AddFailure(directory.FullName, E); + cLogManager.LogEntry($"NTFS subtree scan failed for '{directory.FullName}'. ReturnedPartialResult=true. Error='{E.Message}'", LogLevels.Warning); cLogManager.LogException(E, LogLevels.Debug); } } @@ -330,6 +339,39 @@ namespace LiamNtfs } } + public class cNtfsEnumerationDiagnostics + { + private const int MaxSamples = 10; + + public cNtfsEnumerationDiagnostics() + { + } + + public cNtfsEnumerationDiagnostics(string rootPath, int depth) + { + RootPath = rootPath ?? string.Empty; + Depth = depth; + } + + public string RootPath { get; private set; } = string.Empty; + public int Depth { get; private set; } + public int FoldersEnumerated { get; set; } + public int EnumerationFailures { get; set; } + public int MetadataFailures { get; set; } + public int SubtreeFailures { get; set; } + public List FailureSamples { get; } = new List(); + + public int TotalFailures => EnumerationFailures + MetadataFailures + SubtreeFailures; + + public void AddFailure(string path, Exception exception) + { + if (FailureSamples.Count >= MaxSamples) + return; + + FailureSamples.Add($"{path}: {exception?.GetType().Name}: {exception?.Message}"); + } + } + public class cNtfsLogonInfo {