Add NTFS data area diagnostics logging

LiamNtfs/C4IT.LIAM.Ntfs.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.DirectoryServices.AccountManagement;
 using System.IO;
 using System.Linq;
@@ -52,6 +53,72 @@ namespace C4IT.LIAM
             public int Level { get; set; } = -1;
         }
 
+        private sealed class cNtfsDataAreaDiagnostics
+        {
+            private const int MaxSamples = 10;
+
+            public int ScanRequests { get; set; }
+            public int FoldersEnumerated { get; set; }
+            public int EnumerationFailures { get; set; }
+            public int MetadataFailures { get; set; }
+            public int SubtreeFailures { get; set; }
+            public int DataAreaCandidates { get; set; }
+            public int IncludedDataAreas { get; set; }
+            public int SkippedByDataAreaRegex { get; set; }
+            public int SkippedByExcludeRule { get; set; }
+            public int SkippedByMissingIncludeRule { get; set; }
+            public int SkippedSubtreesByExcludeRule { get; set; }
+            public int SkippedSubtreesByMissingIncludeRule { get; set; }
+            public int AclReadFailures { get; set; }
+            public int AclEntriesEvaluated { get; set; }
+            public int UnresolvedAclSids { get; set; }
+            public int AclGroupsWithoutNamingMatch { get; set; }
+            public int DataAreasWithoutPermissionMapping { get; set; }
+            public List<string> EnumerationFailureSamples { get; } = new List<string>();
+            public List<string> FilterSamples { get; } = new List<string>();
+            public List<string> MappingIssueSamples { get; } = new List<string>();
+
+            public void MergeEnumeration(cNtfsEnumerationDiagnostics diagnostics)
+            {
+                if (diagnostics == null)
+                    return;
+
+                ScanRequests++;
+                FoldersEnumerated += diagnostics.FoldersEnumerated;
+                EnumerationFailures += diagnostics.EnumerationFailures;
+                MetadataFailures += diagnostics.MetadataFailures;
+                SubtreeFailures += diagnostics.SubtreeFailures;
+                AddSamples(EnumerationFailureSamples, diagnostics.FailureSamples);
+            }
+
+            public void AddFilterSample(string sample)
+            {
+                AddSample(FilterSamples, sample);
+            }
+
+            public void AddMappingIssueSample(string sample)
+            {
+                AddSample(MappingIssueSamples, sample);
+            }
+
+            private static void AddSamples(List<string> target, IEnumerable<string> samples)
+            {
+                if (target == null || samples == null)
+                    return;
+
+                foreach (var sample in samples)
+                    AddSample(target, sample);
+            }
+
+            private static void AddSample(List<string> target, string sample)
+            {
+                if (target == null || string.IsNullOrWhiteSpace(sample) || target.Count >= MaxSamples)
+                    return;
+
+                target.Add(sample);
+            }
+        }
+
         public static Guid nftsModuleId = new Guid("77e213a1-6517-ea11-4881-000c2980fd94");
         private const string AdditionalConfigurationExcludePathsKey = "NtfsExcludePaths";
         private const string AdditionalConfigurationIncludePathsKey = "NtfsIncludePaths";
@@ -63,6 +130,7 @@ namespace C4IT.LIAM
         public readonly cActiveDirectoryBase activeDirectoryBase = new cActiveDirectoryBase();
         private readonly Dictionary<string, HashSet<string>> publishedShareCache = new Dictionary<string, HashSet<string>>(StringComparer.OrdinalIgnoreCase);
         private readonly Dictionary<string, string> dfsEntryPathCache = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        private cNtfsDataAreaDiagnostics currentDataAreaDiagnostics;
 
         //public readonly bool WithoutPrivateFolders = true;
 
@@ -163,6 +231,8 @@ namespace C4IT.LIAM
         {
             var CM = MethodBase.GetCurrentMethod();
             LogMethodBegin(CM);
+            var stopwatch = Stopwatch.StartNew();
+            currentDataAreaDiagnostics = new cNtfsDataAreaDiagnostics();
             try
             {
                 if (!cC4ITLicenseM42ESM.Instance.IsValid || !cC4ITLicenseM42ESM.Instance.Modules.ContainsKey(nftsModuleId))
@@ -171,22 +241,37 @@ namespace C4IT.LIAM
                     return new List<cLiamDataAreaBase>();
                 }
                 if (!await LogonAsync())
+                {
+                    LogEntry($"NTFS getDataAreas failed. Stage=Logon RootPath='{RootPath}', Error='{GetLastErrorMessage()}'", LogLevels.Warning);
                     return null;
+                }
                 if (string.IsNullOrEmpty(this.RootPath))
+                {
+                    LogEntry("NTFS getDataAreas failed. Stage=Configuration RootPath is empty.", LogLevels.Warning);
                     return null;
+                }
 
+                LogNtfsAdContext();
                 var DataAreas = new List<cLiamDataAreaBase>();
                 var rootClassification = ClassifyPath(this.RootPath);
                 var rootDataArea = await BuildDataAreaAsync(rootClassification);
                 if (rootDataArea == null)
+                {
+                    LogEntry($"NTFS getDataAreas failed. Stage=RootClassification RootPath='{RootPath}', PathKind='{rootClassification?.Kind}', NormalizedPath='{rootClassification?.NormalizedPath}'", LogLevels.Warning);
                     return null;
+                }
 
                 DataAreas.Add(rootDataArea);
+                currentDataAreaDiagnostics.IncludedDataAreas++;
 
                 if (Depth == 0)
+                {
+                    LogDataAreaScanSummary(DataAreas, stopwatch.Elapsed);
                     return DataAreas;
+                }
 
                 DataAreas.AddRange(await GetChildDataAreasAsync(rootClassification, Depth));
+                LogDataAreaScanSummary(DataAreas, stopwatch.Elapsed);
                 return DataAreas;
             }
             catch (Exception E)
@@ -399,8 +484,12 @@ namespace C4IT.LIAM
             }
 
             var folderEntries = await ntfsBase.RequestFoldersListAsync(parentClassification.NormalizedPath, 1);
+            currentDataAreaDiagnostics?.MergeEnumeration(ntfsBase.LastEnumerationDiagnostics);
             if (folderEntries == null)
+            {
+                LogEntry($"NTFS scan returned null while enumerating children of '{parentClassification.NormalizedPath}'. ChildrenSkipped=true.", LogLevels.Warning);
                 return children;
+            }
 
             foreach (var entry in folderEntries.Values.OfType<cNtfsResultFolder>())
             {
@@ -439,23 +528,47 @@ namespace C4IT.LIAM
             if (classification == null)
                 return false;
 
+            if (currentDataAreaDiagnostics != null)
+                currentDataAreaDiagnostics.DataAreaCandidates++;
+
             if (!MatchesDataAreaRegEx(classification.DisplayName))
+            {
+                if (currentDataAreaDiagnostics != null)
+                {
+                    currentDataAreaDiagnostics.SkippedByDataAreaRegex++;
+                    currentDataAreaDiagnostics.AddFilterSample($"DataAreaRegEx rejected '{classification.NormalizedPath}' with DisplayName='{classification.DisplayName}' and DataAreaRegEx='{this.DataAreaRegEx}'");
+                }
+                LogEntry($"Skip NTFS path '{classification.NormalizedPath}' because DisplayName='{classification.DisplayName}' does not match DataAreaRegEx='{this.DataAreaRegEx}'", LogLevels.Debug);
                 return false;
+            }
 
             string matchingConfigurationKey;
             string matchingRule;
             if (IsPathBlacklisted(classification, out matchingConfigurationKey, out matchingRule))
             {
+                if (currentDataAreaDiagnostics != null)
+                {
+                    currentDataAreaDiagnostics.SkippedByExcludeRule++;
+                    currentDataAreaDiagnostics.AddFilterSample($"Exclude rejected '{classification.NormalizedPath}' via {matchingConfigurationKey}={matchingRule}");
+                }
                 LogEntry($"Skip NTFS path '{classification.NormalizedPath}' due to AdditionalConfiguration rule '{matchingConfigurationKey}={matchingRule}'", LogLevels.Debug);
                 return false;
             }
 
             if (!IsPathWhitelisted(classification, true, out matchingConfigurationKey, out matchingRule))
             {
+                if (currentDataAreaDiagnostics != null)
+                {
+                    currentDataAreaDiagnostics.SkippedByMissingIncludeRule++;
+                    currentDataAreaDiagnostics.AddFilterSample($"Include missing for '{classification.NormalizedPath}' via {AdditionalConfigurationIncludePathsKey}");
+                }
                 LogEntry($"Skip NTFS path '{classification.NormalizedPath}' because no AdditionalConfiguration whitelist matched", LogLevels.Debug);
                 return false;
             }
 
+            if (currentDataAreaDiagnostics != null)
+                currentDataAreaDiagnostics.IncludedDataAreas++;
+
             return true;
         }
 
@@ -468,6 +581,11 @@ namespace C4IT.LIAM
             string matchingRule;
             if (IsPathBlacklisted(classification, out matchingConfigurationKey, out matchingRule))
             {
+                if (currentDataAreaDiagnostics != null)
+                {
+                    currentDataAreaDiagnostics.SkippedSubtreesByExcludeRule++;
+                    currentDataAreaDiagnostics.AddFilterSample($"Subtree exclude rejected '{classification.NormalizedPath}' via {matchingConfigurationKey}={matchingRule}");
+                }
                 LogEntry($"Skip NTFS subtree '{classification.NormalizedPath}' due to AdditionalConfiguration rule '{matchingConfigurationKey}={matchingRule}'", LogLevels.Debug);
                 return false;
             }
@@ -478,6 +596,11 @@ namespace C4IT.LIAM
             if (IsPathWhitelisted(classification, true, out matchingConfigurationKey, out matchingRule))
                 return true;
 
+            if (currentDataAreaDiagnostics != null)
+            {
+                currentDataAreaDiagnostics.SkippedSubtreesByMissingIncludeRule++;
+                currentDataAreaDiagnostics.AddFilterSample($"Subtree outside include whitelist '{classification.NormalizedPath}' via {AdditionalConfigurationIncludePathsKey}");
+            }
             LogEntry($"Skip NTFS subtree '{classification.NormalizedPath}' because it is outside AdditionalConfiguration whitelist '{AdditionalConfigurationIncludePathsKey}'", LogLevels.Debug);
             return false;
         }
@@ -681,6 +804,99 @@ namespace C4IT.LIAM
             return Regex.IsMatch(valueSegment, regexPattern, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
         }
 
+        private void LogNtfsAdContext()
+        {
+            var includePaths = string.Join(";", GetAdditionalConfigurationValues(AdditionalConfigurationIncludePathsKey));
+            var excludePaths = string.Join(";", GetAdditionalConfigurationValues(AdditionalConfigurationExcludePathsKey));
+            var dc = activeDirectoryBase?.EffectiveDomainController;
+
+            LogEntry(
+                $"NTFS getDataAreas context: Stage=Start RootPath='{RootPath}', MaxDepth={MaxDepth}, Domain='{Domain}', DomainController='{(string.IsNullOrWhiteSpace(dc) ? "(domain locator)" : dc)}', Credential='{Credential?.Identification}', GroupStrategy='{GroupStrategy}', DataAreaRegEx='{DataAreaRegEx}', IncludePaths='{includePaths}', ExcludePaths='{excludePaths}'",
+                LogLevels.Info);
+        }
+
+        private void LogDataAreaScanSummary(ICollection<cLiamDataAreaBase> dataAreas, TimeSpan duration)
+        {
+            var diagnostics = currentDataAreaDiagnostics;
+            if (diagnostics == null)
+                return;
+
+            var returnedDataAreas = dataAreas?.Count ?? 0;
+            var permissionDataAreas = dataAreas?.OfType<cLiamNtfsPermissionDataAreaBase>().Count() ?? 0;
+            var issueCount =
+                diagnostics.EnumerationFailures
+                + diagnostics.MetadataFailures
+                + diagnostics.SubtreeFailures
+                + diagnostics.AclReadFailures
+                + diagnostics.UnresolvedAclSids
+                + diagnostics.AclGroupsWithoutNamingMatch
+                + diagnostics.DataAreasWithoutPermissionMapping;
+
+            LogEntry(
+                $"NTFS getDataAreas summary: Stage=Summary RootPath='{RootPath}', MaxDepth={MaxDepth}, ReturnedDataAreas={returnedDataAreas}, PermissionDataAreas={permissionDataAreas}, ScanRequests={diagnostics.ScanRequests}, FoldersEnumerated={diagnostics.FoldersEnumerated}, DataAreaCandidates={diagnostics.DataAreaCandidates}, IncludedCandidates={diagnostics.IncludedDataAreas}, SkippedByDataAreaRegEx={diagnostics.SkippedByDataAreaRegex}, SkippedByExcludeRule={diagnostics.SkippedByExcludeRule}, SkippedByMissingIncludeRule={diagnostics.SkippedByMissingIncludeRule}, SkippedSubtreesByExcludeRule={diagnostics.SkippedSubtreesByExcludeRule}, SkippedSubtreesByMissingIncludeRule={diagnostics.SkippedSubtreesByMissingIncludeRule}, EnumerationFailures={diagnostics.EnumerationFailures}, MetadataFailures={diagnostics.MetadataFailures}, SubtreeFailures={diagnostics.SubtreeFailures}, AclReadFailures={diagnostics.AclReadFailures}, AclEntriesEvaluated={diagnostics.AclEntriesEvaluated}, UnresolvedAclSids={diagnostics.UnresolvedAclSids}, AclGroupsWithoutNamingMatch={diagnostics.AclGroupsWithoutNamingMatch}, DataAreasWithoutPermissionMapping={diagnostics.DataAreasWithoutPermissionMapping}, Duration='{duration}'",
+                issueCount > 0 ? LogLevels.Warning : LogLevels.Info);
+
+            if (diagnostics.EnumerationFailureSamples.Count > 0)
+                LogEntry($"NTFS diagnostic samples: Stage=Enumeration Samples='{string.Join(" | ", diagnostics.EnumerationFailureSamples)}'", LogLevels.Warning);
+
+            if (diagnostics.FilterSamples.Count > 0)
+                LogEntry($"NTFS diagnostic samples: Stage=Filter Samples='{string.Join(" | ", diagnostics.FilterSamples)}'", LogLevels.Info);
+
+            if (diagnostics.MappingIssueSamples.Count > 0)
+                LogEntry($"NTFS diagnostic samples: Stage=AclMapping Samples='{string.Join(" | ", diagnostics.MappingIssueSamples)}'", LogLevels.Warning);
+        }
+
+        internal void RecordAclReadFailure(string path)
+        {
+            if (currentDataAreaDiagnostics != null)
+            {
+                currentDataAreaDiagnostics.AclReadFailures++;
+                currentDataAreaDiagnostics.AddMappingIssueSample($"ACL read failed for '{path}'");
+            }
+
+            LogEntry($"NTFS data area ACL read failed. Stage=AclRead Path='{path}'", LogLevels.Warning);
+        }
+
+        internal void RecordAclEntryEvaluated()
+        {
+            if (currentDataAreaDiagnostics != null)
+                currentDataAreaDiagnostics.AclEntriesEvaluated++;
+        }
+
+        internal void RecordUnresolvedAclSid(string path, string sid)
+        {
+            if (currentDataAreaDiagnostics != null)
+            {
+                currentDataAreaDiagnostics.UnresolvedAclSids++;
+                currentDataAreaDiagnostics.AddMappingIssueSample($"Unresolved ACL SID '{sid}' on '{path}'");
+            }
+
+            LogEntry($"NTFS ACL SID could not be resolved to an AD group. Stage=AclSidResolution Path='{path}', Sid='{sid}'", LogLevels.Warning);
+        }
+
+        internal void RecordAclGroupWithoutNamingMatch(string path, string samAccountName)
+        {
+            if (currentDataAreaDiagnostics != null)
+            {
+                currentDataAreaDiagnostics.AclGroupsWithoutNamingMatch++;
+                currentDataAreaDiagnostics.AddMappingIssueSample($"ACL group '{samAccountName}' on '{path}' did not match naming conventions");
+            }
+        }
+
+        internal void RecordDataAreaWithoutPermissionMapping(string path, IEnumerable<string> aclGroups, string ownerWildcard, string writeWildcard, string readWildcard)
+        {
+            var groups = string.Join(",", aclGroups ?? Enumerable.Empty<string>());
+            if (currentDataAreaDiagnostics != null)
+            {
+                currentDataAreaDiagnostics.DataAreasWithoutPermissionMapping++;
+                currentDataAreaDiagnostics.AddMappingIssueSample($"DataArea '{path}' has no Owner/Write/Read mapping. ACL groups: {groups}");
+            }
+
+            LogEntry(
+                $"NTFS data area returned without Owner/Write/Read group mapping. Stage=AclMapping Path='{path}', AclGroups='{groups}', OwnerWildcard='{ownerWildcard}', WriteWildcard='{writeWildcard}', ReadWildcard='{readWildcard}'",
+                LogLevels.Warning);
+        }
+
         private List<string> GetDfsObjectPrefixes(string path)
         {
             var normalizedPath = NormalizeUncPath(path);
@@ -1347,22 +1563,30 @@ namespace C4IT.LIAM
         {
             var ACLs = Provider.activeDirectoryBase.GetAccessControlList(path);
             if (ACLs == null)
+            {
+                Provider.RecordAclReadFailure(path);
                 return;
+            }
 
             var ownerNamingConvention = Provider.NamingConventions.First(i => i.AccessRole == eLiamAccessRoles.Owner && (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGP && i.Scope == eLiamAccessRoleScopes.Global || Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP && i.Scope == eLiamAccessRoleScopes.DomainLocal));
             var writeNamingConvention = Provider.NamingConventions.First(i => i.AccessRole == eLiamAccessRoles.Write && (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGP && i.Scope == eLiamAccessRoleScopes.Global || Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP && i.Scope == eLiamAccessRoleScopes.DomainLocal));
             var readNamingConvention = Provider.NamingConventions.First(i => i.AccessRole == eLiamAccessRoles.Read && (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGP && i.Scope == eLiamAccessRoleScopes.Global || Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP && i.Scope == eLiamAccessRoleScopes.DomainLocal));
             var traverseNamingConvention = Provider.NamingConventions.First(i => i.AccessRole == eLiamAccessRoles.Traverse);
+            var resolvedAclGroups = new List<string>();
+            var matchedOwner = false;
+            var matchedWrite = false;
+            var matchedRead = false;
             foreach (FileSystemAccessRule rule in ACLs)
             {
                 var aclSid = rule.IdentityReference.Value;
                 if (aclSid == "S-1-1-0")
                     continue;
 
+                Provider.RecordAclEntryEvaluated();
                 GroupPrincipal grp = GroupPrincipal.FindByIdentity(Provider.activeDirectoryBase.adContext, IdentityType.Sid, aclSid);
                 if (grp == null)
                 {
-                    DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' could not be resolved to an AD group. Naming convention matching skipped.");
+                    Provider.RecordUnresolvedAclSid(path, aclSid);
                     continue;
                 }
 
@@ -1373,8 +1597,10 @@ namespace C4IT.LIAM
                     continue;
                 }
 
+                resolvedAclGroups.Add(samAccountName);
                 if (Regex.IsMatch(samAccountName, ownerNamingConvention.Wildcard, RegexOptions.IgnoreCase))
                 {
+                    matchedOwner = true;
                     this.OwnerGroupIdentifier = aclSid;
                     DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' resolved to '{samAccountName}' and matched Owner naming convention '{ownerNamingConvention.Wildcard}'.");
                     if (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP)
@@ -1401,6 +1627,7 @@ namespace C4IT.LIAM
                 }
                 else if (Regex.IsMatch(samAccountName, writeNamingConvention.Wildcard, RegexOptions.IgnoreCase))
                 {
+                    matchedWrite = true;
                     this.WriteGroupIdentifier = aclSid;
                     DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' resolved to '{samAccountName}' and matched Write naming convention '{writeNamingConvention.Wildcard}'.");
                     if (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP)
@@ -1427,6 +1654,7 @@ namespace C4IT.LIAM
                 }
                 else if (Regex.IsMatch(samAccountName, readNamingConvention.Wildcard, RegexOptions.IgnoreCase))
                 {
+                    matchedRead = true;
                     this.ReadGroupIdentifier = aclSid;
                     DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' resolved to '{samAccountName}' and matched Read naming convention '{readNamingConvention.Wildcard}'.");
                     if (Provider.GroupStrategy == eLiamGroupStrategies.Ntfs_AGDLP)
@@ -1458,9 +1686,18 @@ namespace C4IT.LIAM
                 }
                 else
                 {
+                    Provider.RecordAclGroupWithoutNamingMatch(path, samAccountName);
                     DefaultLogger.LogEntry(LogLevels.Debug, $"ACL SID '{aclSid}' on '{path}' resolved to '{samAccountName}', but did not match Owner/Write/Read/Traverse naming conventions.");
                 }
             }
+
+            if (!matchedOwner && !matchedWrite && !matchedRead)
+                Provider.RecordDataAreaWithoutPermissionMapping(
+                    path,
+                    resolvedAclGroups,
+                    ownerNamingConvention.Wildcard,
+                    writeNamingConvention.Wildcard,
+                    readNamingConvention.Wildcard);
         }
     }
 

LiamNtfs/cNtfsBase.cs

@@ -25,6 +25,7 @@ namespace LiamNtfs
         public PrincipalContext adContext = null;
         public Exception LastException { get; private set; } = null;
         public string LastErrorMessage { get; private set; } = null;
+        public cNtfsEnumerationDiagnostics LastEnumerationDiagnostics { get; private set; } = new cNtfsEnumerationDiagnostics();
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void ResetError()
@@ -169,6 +170,7 @@ namespace LiamNtfs
         public async Task<cNtfsCollectionBase> RequestFoldersListAsync(string rootPath, int depth)
         {
             ResetError();
+            LastEnumerationDiagnostics = new cNtfsEnumerationDiagnostics(rootPath, depth);
             try
             {
                 await Task.Delay(0);
@@ -217,7 +219,9 @@ namespace LiamNtfs
                 }
                 catch (Exception E)
                 {
-                    cLogManager.LogEntry($"Could not enumerate directories under '{rootPath.FullName}': {E.Message}", LogLevels.Warning);
+                    LastEnumerationDiagnostics.EnumerationFailures++;
+                    LastEnumerationDiagnostics.AddFailure(rootPath.FullName, E);
+                    cLogManager.LogEntry($"NTFS enumeration failed under '{rootPath.FullName}'. ReturnedPartialResult=true. Error='{E.Message}'", LogLevels.Warning);
                     cLogManager.LogException(E, LogLevels.Debug);
                     return folders;
                 }
@@ -239,11 +243,14 @@ namespace LiamNtfs
                     }
                     catch (Exception E)
                     {
-                        cLogManager.LogEntry($"Could not read directory metadata for '{directory.FullName}': {E.Message}", LogLevels.Warning);
+                        LastEnumerationDiagnostics.MetadataFailures++;
+                        LastEnumerationDiagnostics.AddFailure(directory.FullName, E);
+                        cLogManager.LogEntry($"NTFS directory metadata read failed for '{directory.FullName}'. DirectorySkipped=true. Error='{E.Message}'", LogLevels.Warning);
                         cLogManager.LogException(E, LogLevels.Debug);
                         continue;
                     }
 
+                    LastEnumerationDiagnostics.FoldersEnumerated++;
                     folders.Add(folder);
                     if (depth <= 0)
                         continue;
@@ -256,7 +263,9 @@ namespace LiamNtfs
                     }
                     catch (Exception E)
                     {
-                        cLogManager.LogEntry($"Could not scan subtree '{directory.FullName}': {E.Message}", LogLevels.Warning);
+                        LastEnumerationDiagnostics.SubtreeFailures++;
+                        LastEnumerationDiagnostics.AddFailure(directory.FullName, E);
+                        cLogManager.LogEntry($"NTFS subtree scan failed for '{directory.FullName}'. ReturnedPartialResult=true. Error='{E.Message}'", LogLevels.Warning);
                         cLogManager.LogException(E, LogLevels.Debug);
                     }
                 }
@@ -330,6 +339,39 @@ namespace LiamNtfs
         }
     }
 
+    public class cNtfsEnumerationDiagnostics
+    {
+        private const int MaxSamples = 10;
+
+        public cNtfsEnumerationDiagnostics()
+        {
+        }
+
+        public cNtfsEnumerationDiagnostics(string rootPath, int depth)
+        {
+            RootPath = rootPath ?? string.Empty;
+            Depth = depth;
+        }
+
+        public string RootPath { get; private set; } = string.Empty;
+        public int Depth { get; private set; }
+        public int FoldersEnumerated { get; set; }
+        public int EnumerationFailures { get; set; }
+        public int MetadataFailures { get; set; }
+        public int SubtreeFailures { get; set; }
+        public List<string> FailureSamples { get; } = new List<string>();
+
+        public int TotalFailures => EnumerationFailures + MetadataFailures + SubtreeFailures;
+
+        public void AddFailure(string path, Exception exception)
+        {
+            if (FailureSamples.Count >= MaxSamples)
+                return;
+
+            FailureSamples.Add($"{path}: {exception?.GetType().Name}: {exception?.Message}");
+        }
+    }
+
 
     public class cNtfsLogonInfo
     {