Add NTFS data area diagnostics logging

2026-06-16 14:47:21 +02:00
parent f1fb5dbc1b
commit 275abe8ebd
2 changed files with 283 additions and 4 deletions
--- a/LiamNtfs/C4IT.LIAM.Ntfs.cs
+++ b/LiamNtfs/C4IT.LIAM.Ntfs.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.DirectoryServices.AccountManagement;
 using System.IO;
 using System.Linq;
@@ -52,6 +53,72 @@ namespace C4IT.LIAM
            public int Level { get; set; } = -1;
        }

+        private sealed class cNtfsDataAreaDiagnostics
+        {
+            private const int MaxSamples = 10;
+
+            public int ScanRequests { get; set; }
+            public int FoldersEnumerated { get; set; }
+            public int EnumerationFailures { get; set; }
+            public int MetadataFailures { get; set; }
+            public int SubtreeFailures { get; set; }
+            public int DataAreaCandidates { get; set; }
+            public int IncludedDataAreas { get; set; }
+            public int SkippedByDataAreaRegex { get; set; }
+            public int SkippedByExcludeRule { get; set; }
+            public int SkippedByMissingIncludeRule { get; set; }
+            public int SkippedSubtreesByExcludeRule { get; set; }
+            public int SkippedSubtreesByMissingIncludeRule { get; set; }
+            public int AclReadFailures { get; set; }
+            public int AclEntriesEvaluated { get; set; }
+            public int UnresolvedAclSids { get; set; }
+            public int AclGroupsWithoutNamingMatch { get; set; }
+            public int DataAreasWithoutPermissionMapping { get; set; }
+            public List<string> EnumerationFailureSamples { get; } = new List<string>();
+            public List<string> FilterSamples { get; } = new List<string>();
+            public List<string> MappingIssueSamples { get; } = new List<string>();
+
+            public void MergeEnumeration(cNtfsEnumerationDiagnostics diagnostics)
+            {
+                if (diagnostics == null)
+                    return;
+
+                ScanRequests++;
+                FoldersEnumerated += diagnostics.FoldersEnumerated;
+                EnumerationFailures += diagnostics.EnumerationFailures;
+                MetadataFailures += diagnostics.MetadataFailures;
+                SubtreeFailures += diagnostics.SubtreeFailures;
+                AddSamples(EnumerationFailureSamples, diagnostics.FailureSamples);
+            }
+
+            public void AddFilterSample(string sample)
+            {
+                AddSample(FilterSamples, sample);
+            }
+
+            public void AddMappingIssueSample(string sample)
+            {
+                AddSample(MappingIssueSamples, sample);
+            }
+
+            private static void AddSamples(List<string> target, IEnumerable<string> samples)
+            {
+                if (target == null || samples == null)
+                    return;
+
+                foreach (var sample in samples)
+                    AddSample(target, sample);
+            }
+
+            private static void AddSample(List<string> target, string sample)
+            {
+                if (target == null || string.IsNullOrWhiteSpace(sample) || target.Count >= MaxSamples)
+                    return;
+
+                target.Add(sample);
+            }
+        }
+
        public static Guid nftsModuleId = new Guid("77e213a1-6517-ea11-4881-000c2980fd94");
        private const string AdditionalConfigurationExcludePathsKey = "NtfsExcludePaths";
        private const string AdditionalConfigurationIncludePathsKey = "NtfsIncludePaths";
@@ -63,6 +130,7 @@ namespace C4IT.LIAM
        public readonly cActiveDirectoryBase activeDirectoryBase = new cActiveDirectoryBase();
        private readonly Dictionary<string, HashSet<string>> publishedShareCache = new Dictionary<string, HashSet<string>>(StringComparer.OrdinalIgnoreCase);
        private readonly Dictionary<string, string> dfsEntryPathCache = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        private cNtfsDataAreaDiagnostics currentDataAreaDiagnostics;

        //public readonly bool WithoutPrivateFolders = true;

@@ -163,6 +231,8 @@ namespace C4IT.LIAM
        {
            var CM = MethodBase.GetCurrentMethod();
            LogMethodBegin(CM);
+            var stopwatch = Stopwatch.StartNew();
+            currentDataAreaDiagnostics = new cNtfsDataAreaDiagnostics();
            try
            {
                if (!cC4ITLicenseM42ESM.Instance.IsValid || !cC4ITLicenseM42ESM.Instance.Modules.ContainsKey(nftsModuleId))
@@ -171,22 +241,37 @@ namespace C4IT.LIAM
                    return new List<cLiamDataAreaBase>();
                }
                if (!await LogonAsync())
+                {
+                    LogEntry($"NTFS getDataAreas failed. Stage=Logon RootPath='{RootPath}', Error='{GetLastErrorMessage()}'", LogLevels.Warning);
                    return null;
+                }
                if (string.IsNullOrEmpty(this.RootPath))
+                {
+                    LogEntry("NTFS getDataAreas failed. Stage=Configuration RootPath is empty.", LogLevels.Warning);
                    return null;
+                }

+                LogNtfsAdContext();
                var DataAreas = new List<cLiamDataAreaBase>();
                var rootClassification = ClassifyPath(this.RootPath);
                var rootDataArea = await BuildDataAreaAsync(rootClassification);
                if (rootDataArea == null)
+                {
+                    LogEntry($"NTFS getDataAreas failed. Stage=RootClassification RootPath='{RootPath}', PathKind='{rootClassification?.Kind}', NormalizedPath='{rootClassification?.NormalizedPath}'", LogLevels.Warning);
                    return null;
+                }

                DataAreas.Add(rootDataArea);
+                currentDataAreaDiagnostics.IncludedDataAreas++;

                if (Depth == 0)
+                {
+                    LogDataAreaScanSummary(DataAreas, stopwatch.Elapsed);
                    return DataAreas;
+                }

                DataAreas.AddRange(await GetChildDataAreasAsync(rootClassification, Depth));
+                LogDataAreaScanSummary(DataAreas, stopwatch.Elapsed);
                return DataAreas;
            }
            catch (Exception E)
@@ -399,8 +484,12 @@ namespace C4IT.LIAM
            }

            var folderEntries = await ntfsBase.RequestFoldersListAsync(parentClassification.NormalizedPath, 1);
+            currentDataAreaDiagnostics?.MergeEnumeration(ntfsBase.LastEnumerationDiagnostics);
            if (folderEntries == null)
+            {
+                LogEntry($"NTFS scan returned null while enumerating children of '{parentClassification.NormalizedPath}'. ChildrenSkipped=true.", LogLevels.Warning);
                return children;
+            }

            foreach (var entry in folderEntries.Values.OfType<cNtfsResultFolder>())
            {
@@ -439,23 +528,47 @@ namespace C4IT.LIAM
            if (classification == null)
                return false;

+            if (currentDataAreaDiagnostics != null)
+                currentDataAreaDiagnostics.DataAreaCandidates++;
+
            if (!MatchesDataAreaRegEx(classification.DisplayName))
+            {
+                if (currentDataAreaDiagnostics != null)
+                {
+                    currentDataAreaDiagnostics.SkippedByDataAreaRegex++;
+                    currentDataAreaDiagnostics.AddFilterSample($"DataAreaRegEx rejected '{classification.NormalizedPath}' with DisplayName='{classification.DisplayName}' and DataAreaRegEx='{this.DataAreaRegEx}'");
+                }
+                LogEntry($"Skip NTFS path '{classification.NormalizedPath}' because DisplayName='{classification.DisplayName}' does not match DataAreaRegEx='{this.DataAreaRegEx}'", LogLevels.Debug);
                return false;
+            }

            string matchingConfigurationKey;
            string matchingRule;
            if (IsPathBlacklisted(classification, out matchingConfigurationKey, out matchingRule))
            {
+                if (currentDataAreaDiagnostics != null)
+                {
+                    currentDataAreaDiagnostics.SkippedByExcludeRule++;
+                    currentDataAreaDiagnostics.AddFilterSample($"Exclude rejected '{classification.NormalizedPath}' via {matchingConfigurationKey}={matchingRule}");
+                }
                LogEntry($"Skip NTFS path '{classification.NormalizedPath}' due to AdditionalConfiguration rule '{matchingConfigurationKey}={matchingRule}'", LogLevels.Debug);
                return false;
            }

            if (!IsPathWhitelisted(classification, true, out matchingConfigurationKey, out matchingRule))
            {
+                if (currentDataAreaDiagnostics != null)
+                {
+                    currentDataAreaDiagnostics.SkippedByMissingIncludeRule++;
+                    currentDataAreaDiagnostics.AddFilterSample($"Include missing for '{classification.NormalizedPath}' via {AdditionalConfigurationIncludePathsKey}");
+                }
                LogEntry($"Skip NTFS path '{classification.NormalizedPath}' because no AdditionalConfiguration whitelist matched", LogLevels.Debug);
                return false;
            }

+            if (currentDataAreaDiagnostics != null)
+                currentDataAreaDiagnostics.IncludedDataAreas++;
+
            return true;
        }

@@ -468,6 +581,11 @@ namespace C4IT.LIAM
            string matchingRule;
            if (IsPathBlacklisted(classification, out matchingConfigurationKey, out matchingRule))
            {
+                if (currentDataAreaDiagnostics != null)
+                {
+                    currentDataAreaDiagnostics.SkippedSubtreesByExcludeRule++;
+                    currentDataAreaDiagnostics.AddFilterSample($"Subtree exclude rejected '{classification.NormalizedPath}' via {matchingConfigurationKey}={matchingRule}");
+                }
                LogEntry($"Skip NTFS subtree '{classification.NormalizedPath}' due to AdditionalConfiguration rule '{matchingConfigurationKey}={matchingRule}'", LogLevels.Debug);
                return false;
            }
@@ -478,6 +596,11 @@ namespace C4IT.LIAM
            if (IsPathWhitelisted(classification, true, out matchingConfigurationKey, out matchingRule))
                return true;

+            if (currentDataAreaDiagnostics != null)
+            {
+                currentDataAreaDiagnostics.SkippedSubtreesByMissingIncludeRule++;
+                currentDataAreaDiagnostics.AddFilterSample($"Subtree outside include whitelist '{classification.NormalizedPath}' via {AdditionalConfigurationIncludePathsKey}");
+            }
            LogEntry($"Skip NTFS subtree '{classification.NormalizedPath}' because it is outside AdditionalConfiguration whitelist '{AdditionalConfigurationIncludePathsKey}'", LogLevels.Debug);
            return false;
        }
@@ -681,6 +804,99 @@ namespace C4IT.LIAM
            return Regex.IsMatch(valueSegment, regexPattern, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
        }

+        private void LogNtfsAdContext()
+        {
+            var includePaths = string.Join(";", GetAdditionalConfigurationValues(AdditionalConfigurationIncludePathsKey));
+            var excludePaths = string.Join(";", GetAdditionalConfigurationValues(AdditionalConfigurationExcludePathsKey));
+            var dc = activeDirectoryBase?.EffectiveDomainController;
+
+            LogEntry(
+                $"NTFS getDataAreas context: Stage=Start RootPath='{RootPath}', MaxDepth={MaxDepth}, Domain='{Domain}', DomainController='{(string.IsNullOrWhiteSpace(dc) ? "(domain locator)" : dc)}', Credential='{Credential?.Identification}', GroupStrategy='{GroupStrategy}', DataAreaRegEx='{DataAreaRegEx}', IncludePaths='{includePaths}', ExcludePaths='{excludePaths}'",
+                LogLevels.Info);
+        }
+
+        private void LogDataAreaScanSummary(ICollection<cLiamDataAreaBase> dataAreas, TimeSpan duration)
+        {
+            var diagnostics = currentDataAreaDiagnostics;
+            if (diagnostics == null)
+                return;
+
+            var returnedDataAreas = dataAreas?.Count ?? 0;
+            var permissionDataAreas = dataAreas?.OfType<cLiamNtfsPermissionDataAreaBase>().Count() ?? 0;
+            var issueCount =
+                diagnostics.EnumerationFailures
+                + diagnostics.MetadataFailures
+                + diagnostics.SubtreeFailures
+                + diagnostics.AclReadFailures
+                + diagnostics.UnresolvedAclSids
+                + diagnostics.AclGroupsWithoutNamingMatch
+                + diagnostics.DataAreasWithoutPermissionMapping;
+
+            LogEntry(
+                $"NTFS getDataAreas summary: Stage=Summary RootPath='{RootPath}', MaxDepth={MaxDepth}, ReturnedDataAreas={returnedDataAreas}, PermissionDataAreas={permissionDataAreas}, ScanRequests={diagnostics.ScanRequests}, FoldersEnumerated={diagnostics.FoldersEnumerated}, DataAreaCandidates={diagnostics.DataAreaCandidates}, IncludedCandidates={diagnostics.IncludedDataAreas}, SkippedByDataAreaRegEx={diagnostics.SkippedByDataAreaRegex}, SkippedByExcludeRule={diagnostics.SkippedByExcludeRule}, SkippedByMissingIncludeRule={diagnostics.SkippedByMissingIncludeRule}, SkippedSubtreesByExcludeRule={diagnostics.SkippedSubtreesByExcludeRule}, SkippedSubtreesByMissingIncludeRule={diagnostics.SkippedSubtreesByMissingIncludeRule}, EnumerationFailures={diagnostics.EnumerationFailures}, MetadataFailures={diagnostics.MetadataFailures}, SubtreeFailures={diagnostics.SubtreeFailures}, AclReadFailures={diagnostics.AclReadFailures}, AclEntriesEvaluated={diagnostics.AclEntriesEvaluated}, UnresolvedAclSids={diagnostics.UnresolvedAclSids}, AclGroupsWithoutNamingMatch={diagnostics.AclGroupsWithoutNamingMatch}, DataAreasWithoutPermissionMapping={diagnostics.DataAreasWithoutPermissionMapping}, Duration='{duration}'",
+                issueCount > 0 ? LogLevels.Warning : LogLevels.Info);
+
+            if (diagnostics.EnumerationFailureSamples.Count > 0)
+                LogEntry($"NTFS diagnostic samples: Stage=Enumeration Samples='{string.Join(" | ", diagnostics.EnumerationFailureSamples)}'", LogLevels.Warning);
+
+            if (diagnostics.FilterSamples.Count > 0)
+                LogEntry($"NTFS diagnostic samples: Stage=Filter Samples='{string.Join(" | ", diagnostics.FilterSamples)}'", LogLevels.Info);
+
+            if (diagnostics.MappingIssueSamples.Count > 0)
+                LogEntry($"NTFS diagnostic samples: Stage=AclMapping Samples='{string.Join(" | ", diagnostics.MappingIssueSamples)}'", LogLevels.Warning);
+        }
+
+        internal void RecordAclReadFailure(string path)
+        {
+            if (currentDataAreaDiagnostics != null)
+            {
+                currentDataAreaDiagnostics.AclReadFailures++;
+                currentDataAreaDiagnostics.AddMappingIssueSample($"ACL read failed for '{path}'");
+            }
+
+            LogEntry($"NTFS data area ACL read failed. Stage=AclRead Path='{path}'", LogLevels.Warning);
+        }
+
+        internal void RecordAclEntryEvaluated()
+        {
+            if (currentDataAreaDiagnostics != null)
+                currentDataAreaDiagnostics.AclEntriesEvaluated++;
+        }
+
+        internal void RecordUnresolvedAclSid(string path, string sid)
+        {
+            if (currentDataAreaDiagnostics != null)
+            {
+                currentDataAreaDiagnostics.UnresolvedAclSids++;
+                currentDataAreaDiagnostics.AddMappingIssueSample($"Unresolved ACL SID '{sid}' on '{path}'");
+            }
+
+            LogEntry($"NTFS ACL SID could not be resolved to an AD group. Stage=AclSidResolution Path='{path}', Sid='{sid}'", LogLevels.Warning);
+        }
+
+        internal void RecordAclGroupWithoutNamingMatch(string path, string samAccountName)
+        {
+            if (currentDataAreaDiagnostics != null)
+            {
+                currentDataAreaDiagnostics.AclGroupsWithoutNamingMatch++;
+                currentDataAreaDiagnostics.AddMappingIssueSample($"ACL group '{samAccountName}' on '{path}' did not match naming conventions");
+            }
+        }
+
+        internal void RecordDataAreaWithoutPermissionMapping(string path, IEnumerable<string> aclGroups, string ownerWildcard, string writeWildcard, string readWildcard)
+        {
+            var groups = string.Join(",", aclGroups ?? Enumerable.Empty<string>());
+            if (currentDataAreaDiagnostics != null)
+            {
+                currentDataAreaDiagnostics.DataAreasWithoutPermissionMapping++;
+                currentDataAreaDiagnostics.AddMappingIssueSample($"DataArea '{path}' has no Owner/Write/Read mapping. ACL groups: {groups}");
+            }
+
+            LogEntry(
+                $"NTFS data area returned without Owner/Write/Read group mapping. Stage=AclMapping Path='{path}', AclGroups='{groups}', OwnerWildcard='{ownerWildcard}', WriteWildcard='{writeWildcard}', ReadWildcard='{readWildcard}'",
+                LogLevels.Warning);
+        }
+
        private List<string> GetDfsObjectPrefixes(string path)
        {
            var normalizedPath = NormalizeUncPath(path);
@@ -1347,22 +1563,30 @@ namespace C4IT.LIAM
        {
            var ACLs = Provider.activeDirectoryBase.GetAccessControlList(path);
            if (ACLs == null)
+            {
+                Provider.RecordAclReadFailure(path);
                return;
+            }

            var ownerNamingConvention = Provider.NamingConventions.First(i => i.AccessRole == eLiamAccessRoles.Owner && (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGP && i.Scope == eLiamAccessRoleScopes.Global || Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP && i.Scope == eLiamAccessRoleScopes.DomainLocal));
            var writeNamingConvention = Provider.NamingConventions.First(i => i.AccessRole == eLiamAccessRoles.Write && (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGP && i.Scope == eLiamAccessRoleScopes.Global || Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP && i.Scope == eLiamAccessRoleScopes.DomainLocal));
            var readNamingConvention = Provider.NamingConventions.First(i => i.AccessRole == eLiamAccessRoles.Read && (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGP && i.Scope == eLiamAccessRoleScopes.Global || Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP && i.Scope == eLiamAccessRoleScopes.DomainLocal));
            var traverseNamingConvention = Provider.NamingConventions.First(i => i.AccessRole == eLiamAccessRoles.Traverse);
+            var resolvedAclGroups = new List<string>();
+            var matchedOwner = false;
+            var matchedWrite = false;
+            var matchedRead = false;
            foreach (FileSystemAccessRule rule in ACLs)
            {
                var aclSid = rule.IdentityReference.Value;
                if (aclSid == "S-1-1-0")
                    continue;

+                Provider.RecordAclEntryEvaluated();
                GroupPrincipal grp = GroupPrincipal.FindByIdentity(Provider.activeDirectoryBase.adContext, IdentityType.Sid, aclSid);
                if (grp == null)
                {
-                    DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' could not be resolved to an AD group. Naming convention matching skipped.");
+                    Provider.RecordUnresolvedAclSid(path, aclSid);
                    continue;
                }

@@ -1373,8 +1597,10 @@ namespace C4IT.LIAM
                    continue;
                }

+                resolvedAclGroups.Add(samAccountName);
                if (Regex.IsMatch(samAccountName, ownerNamingConvention.Wildcard, RegexOptions.IgnoreCase))
                {
+                    matchedOwner = true;
                    this.OwnerGroupIdentifier = aclSid;
                    DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' resolved to '{samAccountName}' and matched Owner naming convention '{ownerNamingConvention.Wildcard}'.");
                    if (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP)
@@ -1401,6 +1627,7 @@ namespace C4IT.LIAM
                }
                else if (Regex.IsMatch(samAccountName, writeNamingConvention.Wildcard, RegexOptions.IgnoreCase))
                {
+                    matchedWrite = true;
                    this.WriteGroupIdentifier = aclSid;
                    DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' resolved to '{samAccountName}' and matched Write naming convention '{writeNamingConvention.Wildcard}'.");
                    if (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP)
@@ -1427,6 +1654,7 @@ namespace C4IT.LIAM
                }
                else if (Regex.IsMatch(samAccountName, readNamingConvention.Wildcard, RegexOptions.IgnoreCase))
                {
+                    matchedRead = true;
                    this.ReadGroupIdentifier = aclSid;
                    DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' resolved to '{samAccountName}' and matched Read naming convention '{readNamingConvention.Wildcard}'.");
                    if (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP)
@@ -1458,9 +1686,18 @@ namespace C4IT.LIAM
                }
                else
                {
+                    Provider.RecordAclGroupWithoutNamingMatch(path, samAccountName);
                    DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' resolved to '{samAccountName}', but did not match Owner/Write/Read/Traverse naming conventions.");
                }
            }
+
+            if (!matchedOwner && !matchedWrite && !matchedRead)
+                Provider.RecordDataAreaWithoutPermissionMapping(
+                    path,
+                    resolvedAclGroups,
+                    ownerNamingConvention.Wildcard,
+                    writeNamingConvention.Wildcard,
+                    readNamingConvention.Wildcard);
        }
    }