using C4IT.FASD.Base;
using C4IT.FASD.Communication.Agent;
using C4IT.Logging;

using Newtonsoft.Json;

using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;

using static C4IT.Logging.cLogManager;

namespace C4IT.DataHistoryProvider
{
    public class cDataHistoryCollectorClientAgent : cDataHistoryCollectorModule
    {
        public const string constConnectorName = "F4SD client agent connector";
        public const string constLicenseId = "F8A3284C-E481-4D08-A921-949D2EBD4E0B";

        private const string constAgentDeviceTableName = "agnt-computer";
        private const string constAgentUserTableName = "agnt-user";
        private const string constAgentEventNumericTableName = "agnt-computer-event-numerical";
        private const string constAgentEventStringTableName = "agnt-computer-event-string";

        private const int constMaxDetailsEventsPerDay = 100;

        public int OrganizationCode { get; private set; } = -1;

        public bool useDailyEventTables { get; private set; } = false;
        public bool useOverallEventTables { get; private set; } = false;

        private readonly cAgentApiHelper _agentApiHelper = new cAgentApiHelper();

        private class jsonTypeSettings
        {
            public string[] values { get; set; } = null;
        }

        private struct structEventTableNames
        {
            public DateTime From;
            public DateTime To;
            public string SimpleValues;
            public string JsonData;
            public string DeviceActivity;
            public string UserActivity;
        }

        private class cActivityEventTiming
        {
            internal int deviceScan = 0;
            internal int deviceSend = 0;
            internal int userScan = 0;
            internal int userSend = 0;
        }

        private class cEventTablesInfo : SortedList<DateTime, structEventTableNames>
        {
            public DateTime LastUpdate = DateTime.MinValue;
        };

        private static class TableNamesCriticalSection
        {
            private static SemaphoreSlim Semaphore = new SemaphoreSlim(1, 1);

            public static async Task EnterAsync()
            {
                await Semaphore.WaitAsync();
            }

            public static void Leave()
            {
                Semaphore.Release();
            }
        }

        private class cMeasureCodeInformation
        {
            internal List<cHistoricAggregationEntry> Entries = new List<cHistoricAggregationEntry>();
            internal bool needsJson = false;

            internal void AddEntry(cHistoricAggregationEntry entry)
            {
                Entries.Add(entry);
                if (entry.Column.SourceJsonColumn >=0 || !string.IsNullOrEmpty(entry.Column.SourceJsonField))
                    needsJson = true;
            }
        }

        private cEventTablesInfo EventTablesInfo = new cEventTablesInfo();

        public override void TablePostConfig(cDataHistoryConfigTable Table)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            try
            {
                var AdDomains = Collector.InfrastructureConfig.ActiveDirectory?.Domains;
                var localAccountAssignments = Collector.InfrastructureConfig.ClientAgent?.localAccountAssignments;
                if (AdDomains != null && localAccountAssignments != null)
                    foreach (var localAccountAssignment in localAccountAssignments)
                        if (AdDomains.TryGetValue(localAccountAssignment.Domain, out var adDomain))
                            localAccountAssignment.DomainDns = adDomain.FQDN.ToLowerInvariant();
            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (CM != null) LogMethodEnd(CM);
            }
        }

        public cDataHistoryCollectorClientAgent(cDataHistoryCollector Collector) : base(Collector, enumDataHistoryOrigin.F4sdAgent, constConnectorName, constLicenseId)
        {
        }

        public async Task<bool> CheckAndInitializeDbAccess(cF4sdWebRequestInfo requestInfo, int LogDeep, CancellationToken Token)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            try
            {
                Collector.DoProcessUiMessage(0, "");
                Collector.DoProcessUiMessage(0, "checking access to the agent database");

                if (!DataHistorySqlHelper.GetWellKnownSqlStatement("GetOrganizationCodeFromAgent", out var Query))
                    return false;

                var AgentDb = Collector.InfrastructureConfig.ClientAgent.Connection;
                using (var ConnAgent = new cDbConnection(AgentDb))
                {
                    if (!ConnAgent.IsOpen)
                    {
                        LogEntry($"Could not open client agent database '{AgentDb.Database}', aborting  query'", LogLevels.Fatal);
                        return false;
                    }

                    var Params = new Dictionary<string, object>() { { "ORGA", Collector.InfrastructureConfig.ClientAgent.Organization } };

                    var _ogc = await DataHistorySqlHelper.GetScalarResultAsync<int?>(ConnAgent, Query, Params, requestInfo, Token);

                    if (_ogc == null)
                    {
                        Collector.DoProcessUiMessage(1, $"Could not find the organization code for '{Collector.InfrastructureConfig.ClientAgent.Organization}'");
                        return false;
                    }

                    OrganizationCode = (int)_ogc;

                    if (DataHistorySqlHelper.GetWellKnownSqlStatement("ValidateOverallAgentEventTables", out Query))
                    {
                        var _n = await DataHistorySqlHelper.GetScalarResultAsync<int?>(ConnAgent, Query, null, requestInfo, Token);
                        if (_n == 2)
                            useOverallEventTables = true;
                    }

                    if (DataHistorySqlHelper.GetWellKnownSqlStatement("ValidateDailyAgentEventTables", out Query))
                    {
                        var _n = await DataHistorySqlHelper.GetScalarResultAsync<int?>(ConnAgent, Query, null, requestInfo, Token);
                        if (_n == 1)
                        {
                            useDailyEventTables = true;
                            await getDailyEventTables(5, requestInfo, LogDeep, Token);
                        }
                    }

                    if (!useDailyEventTables && !useOverallEventTables)
                    {
                        Collector.DoProcessUiMessage(1, "Could not find a valid combination of event tables.'");
                        LogEntry($"can neither find the daily tables (c4it_event_mapping) nor the total tables (c4it_event, c4it_event_data) for the events", LogLevels.Fatal);
                        return false;
                    }


                    Collector.DoProcessUiMessage(0, "the agent database access is valid.");
                    return true;
                }
            }
            catch (Exception E)
            {
                Collector.DoProcessUiMessage(0, "undefined error while validating agent database access.");
                LogException(E);
            }
            finally
            {
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }

            return false;
        }

        public override async Task<bool> DoScanAsync(bool Always, bool Rescan, cF4sdWebRequestInfo requestInfo, int LogDeep, CancellationToken Token, bool EnhancedDebug = false)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            var ShowFinished = false;
            try
            {
                Collector.DoProcessUiMessage(0, "");

                var scanInfo = await Collector.F4sdAgent.GetScanTimeInfoAsync(requestInfo, LogDeep + 1, Token);
                if (scanInfo == null)
                {
                    Collector.DoProcessUiMessage(0, "could not get valid scan times info for F4SD agent");
                    return false;
                }
                if (scanInfo.NextScan == null)
                {
                    Collector.DoProcessUiMessage(0, "not valid next scan time info for F4SD agent");
                    return false;
                }

                if (!Always)
                {
                    var nextScan = (DateTime)scanInfo.NextScan;
                    if (!(scanInfo.LastScan is DateTime lastScan) || (lastScan >= nextScan))
                    {
                        Collector.DoProcessUiMessage(0, $"currently no scan needed for F4SD agent, lastScan={scanInfo.LastScan} (UTC), nextScan={scanInfo.NextScan} (UTC)");
                        return false;
                    }
                }

                Collector.DoProcessUiMessage(0, "start doing the agent database scan...");
                Collector.DoProcessUiMessage(0, "");
                ShowFinished = true;

                var _scanStartTime = DateTime.UtcNow;

                var AgentDb = Collector.InfrastructureConfig.ClientAgent.Connection;
                using (var ConnDb = new cDbConnection(Collector.mainDbConnection))
                using (var ConnAgent = new cDbConnection(AgentDb))
                {
                    if (!ConnDb.IsOpen)
                    {
                        LogEntry($"Could not open main sql database '{Collector.mainDbConnection.Database}', aborting scan...'", LogLevels.Error);
                        return false;
                    }
                    if (!ConnAgent.IsOpen)
                    {
                        LogEntry($"Could not open client agent database '{AgentDb.Database}', aborting scan...'", LogLevels.Error);
                        return false;
                    }

                    var RetVal = true;

                    if (!await ScanDevicesAsync(ConnDb, ConnAgent, scanInfo.LastScan, requestInfo, LogDeep + 1, Token))
                    {
                        LogEntry($"Error while scanning the devices from the client agent database, aborting scan...'", LogLevels.Error);
                        RetVal = false;
                    }

                    if (!await ScanAccountsAsync(ConnDb, ConnAgent, scanInfo.LastScan, requestInfo, LogDeep + 1, Token))
                    {
                        LogEntry($"Error while scanning the device accounts from the client agent database, aborting scan...'", LogLevels.Error);
                        RetVal = false;
                    }

                    if (RetVal)
                        await Collector.SetLastScanTime("AgentScan-all", _scanStartTime, requestInfo, LogDeep + 1, Token);

                    return RetVal;
                }

            }
            catch (Exception E)
            {
                Collector.DoProcessUiMessage(0, "Error while database scan!");
                LogException(E);
            }
            finally
            {
                Collector.DoProcessUiMessage(0, "");
                if (ShowFinished)
                    Collector.DoProcessUiMessage(0, "Agent database scan finished.");
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);

            }

            return false;
        }

        private async Task RemoveClientWithoutAgentIDAsync(cDbConnection DBConn, cF4sdWebRequestInfo requestInfo, int LogDeep, CancellationToken Token)
        {

            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }

            var apiError = 0;

            if (!DataHistorySqlHelper.GetWellKnownSqlStatement("RemoveClientsWithoutID", out var Query))
                return;

            var _startTime = DateTime.UtcNow;
            try
            {

                var n = await DataHistorySqlHelper.ExecuteAsync(DBConn, Query, null, requestInfo, Token);

                if (n is null) n = 0;
                Collector.DoProcessUiMessage(2, $"{n} devices removed, because identification changed.");

            }
            catch (Exception E)
            {
                apiError = E.HResult;
                LogException(E);
            }
            finally
            {
                if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(Query.Name, _startTime, "", apiError, requestInfo?.requestName);
                if (CM != null) LogMethodEnd(CM);
            }
        }

        private async Task<bool> ScanDevicesDailyAsync(cDbConnection DBConn, cDbConnection AgentConn, string EventTable, DateTime lastScanTime, cF4sdWebRequestInfo requestInfo, int LogDeep, CancellationToken Token)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }

            var apiError = 0;
            try
            {
                cDbQuery QueryGet;
                if (EventTable == null)
                {
                    if (!DataHistorySqlHelper.GetWellKnownSqlStatement("GetAgentDevicesTotal", out QueryGet))
                        return false;
                }
                else
                {
                    if (!DataHistorySqlHelper.GetWellKnownSqlStatement("GetAgentDevicesDaily", out var QueryGet2))
                        return false;

                    var _strQuery = string.Format(QueryGet2.Query, EventTable);

                    QueryGet = QueryGet2.Clone(_strQuery);
                }

                if (!DataHistorySqlHelper.GetWellKnownSqlStatement("InsertAgentDevice", out var QueryInsert))
                    return false;

                var SqlParams = new Dictionary<string, object>() { { "LASTSCAN", lastScanTime } };

                var Count = 0;
                var Updated = 0;
                var RecCount = 0;
                var found = false;

                double _duration = 0;
                var _startTime2 = DateTime.UtcNow;
                var _startTime = _startTime2;

                try
                {
                    using (var Reader = await DataHistorySqlHelper.GetTableResultAsync(AgentConn, QueryGet, SqlParams, Token))
                    {
                        _duration = (DateTime.UtcNow - _startTime2).TotalMilliseconds;
                        _startTime2 = DateTime.UtcNow;

                        if (Reader != null)
                        {
                            while (await Reader.ReadAsync())
                            {
                                _duration += (DateTime.UtcNow - _startTime2).TotalMilliseconds;
                                try
                                {
                                    RecCount++;
                                    var id = Reader.GetInt32(0);
                                    var name = Reader.IsDBNull(1) ? null : Reader.GetFieldValue<string>(1);
                                    var domainDns = Reader.IsDBNull(2) ? null : Reader.GetFieldValue<string>(2).ToLowerInvariant();
                                    if (string.IsNullOrWhiteSpace(name))
                                        continue;


                                    var lastUpdate = Reader.IsDBNull(3) ? (DateTime?)null : Reader.GetDateTime(3);
                                    var lastUpdate2 = Reader.IsDBNull(4) ? (DateTime?)null : Reader.GetDateTime(4);
                                    if (lastUpdate == null || (lastUpdate2 != null && lastUpdate2 > lastUpdate))
                                        lastUpdate = lastUpdate2;

                                    var tenantId = Reader.IsDBNull(5) ? null : Reader.GetString(5);

                                    name = name.ToUpperInvariant();

                                    string upn = null;
                                    string domainNB = null;
                                    if (!string.IsNullOrWhiteSpace(domainDns))
                                    {
                                        upn = name.ToLowerInvariant() + "$@" + domainDns.ToLowerInvariant();
                                        domainNB = await Collector.ActiveDirectory.GetDomainNB(domainDns, requestInfo, LogDeep + 1, Token);
                                    }
                                    else if (!string.IsNullOrEmpty(tenantId))
                                    {
                                        if (tenantId.StartsWith("\"") && tenantId.EndsWith("\""))
                                            tenantId = tenantId.Substring(1, tenantId.Length - 2);
                                        upn = name.ToLowerInvariant() + "$@" + tenantId;
                                    }
                                    else
                                    {
                                        upn = name.ToLowerInvariant();
                                        domainDns = null;
                                    }

                                    var lastSeen = DateTime.UtcNow;
                                    var SqlParams2 = new Dictionary<string, object>()
                                {
                                    { "id_agent" , id},
                                    { "name" , name},
                                    { "upn_internal" , upn},
                                    { "last_seen" , lastSeen},
                                    { "lastAgent", lastUpdate },
                                    { "domainDns", domainDns },
                                    { "domainNB", domainNB }
                                };

                                    var ret = await DataHistorySqlHelper.ExecuteAsync(DBConn, QueryInsert, SqlParams2, requestInfo, Token);
                                    if (ret >= 0)
                                        Count++;
                                    if (ret > 0)
                                        Updated++;
                                }
                                catch (Exception E)
                                {
                                    LogException(E);
                                }
                            }

                            _duration += (DateTime.UtcNow - _startTime2).TotalMilliseconds;

                            found = true;
                        }
                    }
                }
                catch (Exception E)
                {
                    apiError = E.HResult;
                    LogException(E);
                }
                finally
                {
                    if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(QueryGet.Name, _startTime, _duration, SqlParams, apiError, requestInfo?.requestName);
                    if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(QueryGet.Name + "(total)", _startTime, SqlParams, apiError, requestInfo?.requestName);
                }

                Collector.DoProcessUiMessage(2, $"{RecCount} devices found, {Count} entries processed, {Updated} inserted or updated.");

                await RemoveClientWithoutAgentIDAsync(DBConn, requestInfo, LogDeep + 1, Token);

                return found;
            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (CM != null) LogMethodEnd(CM);
            }

            return false;

        }

        private async Task<bool> ScanDevicesAsync(cDbConnection DBConn, cDbConnection AgentConn, DateTime lastScanTime, cF4sdWebRequestInfo requestInfo, int LogDeep, CancellationToken Token)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            try
            {
                Collector.DoProcessUiMessage(1, "Start scanning the static device properties...");

                var _eventTiming = await getActivityEventTiming(Token, requestInfo, LogDeep + 1);
                var _scanTime = lastScanTime - TimeSpan.FromMilliseconds(_eventTiming.deviceScan + _eventTiming.deviceSend) - TimeSpan.FromHours(1);

                bool _retVal = true;

                if (useOverallEventTables)
                {
                    if (!(EventTablesInfo?.Count >= 1) || EventTablesInfo.First().Value.From > _scanTime)
                        _retVal &= await ScanDevicesDailyAsync(DBConn, AgentConn, null, _scanTime, requestInfo, LogDeep + 1, Token);
                }

                if (useDailyEventTables)
                {
                    // get the list of daily event tables
                    var _listTables = await getDailyEventTables(_scanTime, DateTime.MaxValue, requestInfo, LogDeep + 1, Token);
                    foreach (var _tableName in _listTables)
                        _retVal &= await ScanDevicesDailyAsync(DBConn, AgentConn, _tableName.DeviceActivity, _scanTime, requestInfo, LogDeep + 1, Token);
                }


                return _retVal;
            }
            catch (Exception E)
            {
                Collector.DoProcessUiMessage(1, "Error while scanning the static device properties!");
                LogException(E);
            }
            finally
            {
                Collector.DoProcessUiMessage(1, "Scanning the static device properties finished.");
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }

            return false;
        }

        private async Task<bool> ScanAccountsDailyAsync(cDbConnection DBConn, cDbConnection AgentConn, string EventTable, DateTime lastScanTime, cF4sdWebRequestInfo requestInfo, int LogDeep, CancellationToken Token)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }

            var apiError = 0;
            try
            {
                cDbQuery QueryGet;
                if (EventTable == null)
                {
                    if (!DataHistorySqlHelper.GetWellKnownSqlStatement("GetAgentAccountsTotal", out QueryGet))
                        return false;
                }
                else
                {
                    if (!DataHistorySqlHelper.GetWellKnownSqlStatement("GetAgentAccountsDaily", out var QueryGet2))
                        return false;

                    var _strQuery = string.Format(QueryGet2.Query, EventTable);

                    QueryGet = QueryGet2.Clone(_strQuery);
                }

                if (!DataHistorySqlHelper.GetWellKnownSqlStatement("UpdateMainUserAgent", out var QueryInsert))
                    return false;
                if (!DataHistorySqlHelper.GetWellKnownSqlStatement("UpdateMainUserAgentAzure", out var QueryInsertAzure))
                    return false;
                if (!DataHistorySqlHelper.GetWellKnownSqlStatement("GetMainUserByAccount", out var QueryGetAdAccount))
                    return false;
                if (!DataHistorySqlHelper.GetWellKnownSqlStatement("InsertLocalUser", out var QueryInsertLocalAccount))
                    return false;
                if (!DataHistorySqlHelper.GetWellKnownSqlStatement("UpdateMainUserLastSeen", out var QueryUpdateMainUserLastSeen))
                    return false;

                var SqlParams = new Dictionary<string, object>() { { "LASTSCAN", lastScanTime } };

                var localAccountAssignments = Collector.InfrastructureConfig.ClientAgent.localAccountAssignments;

                var Count = 0;
                var Updated = 0;
                var RecCount = 0;
                var found = false;

                double _duration = 0;
                var _startTime2 = DateTime.UtcNow;
                var _startTime = _startTime2;

                try
                {
                    using (var Reader = await DataHistorySqlHelper.GetTableResultAsync(AgentConn, QueryGet, SqlParams, Token))
                    {
                        _duration = (DateTime.UtcNow - _startTime2).TotalMilliseconds;
                        _startTime2 = DateTime.UtcNow;
                        if (Reader != null)
                        {
                            while (await Reader.ReadAsync())
                            {
                                _duration += (DateTime.UtcNow - _startTime2).TotalMilliseconds;
                                try
                                {
                                    var id = Reader.GetInt32(0);
                                    var strSid = Reader.GetString(1);
                                    var name = Reader.GetFieldValue<string>(2);
                                    var domain = Reader.IsDBNull(3) ? null : Reader.GetFieldValue<string>(3);
                                    var lastAgent = Reader.IsDBNull(4) ? (DateTime?)null : Reader.GetDateTime(4);

                                    if (localAccountAssignments == null && string.IsNullOrEmpty(domain))
                                        continue;

                                    var objSid = cDataHistoryCollectorActiveDirectory.GetByteArrayFromSid(strSid);
                                    if (objSid == null)
                                        continue;

                                    // check if wwe have a local account and process the localAccountAssignments
                                    if (string.IsNullOrEmpty(domain) && localAccountAssignments != null)
                                    {
                                        var AdId = Guid.Empty;
                                        foreach (var localAccountAssignment in localAccountAssignments)
                                        {
                                            var searchAccount = name;
                                            if (!string.IsNullOrEmpty(localAccountAssignment.RegExFilter))
                                                searchAccount = Regex.Match(searchAccount, localAccountAssignment.RegExFilter).Value;
                                            if (!string.IsNullOrEmpty(localAccountAssignment.AccountMask))
                                                searchAccount = localAccountAssignment.AccountMask.Replace("*", searchAccount);

                                            var Params = new Dictionary<string, object>() { { "Account", searchAccount }, { "Domain", localAccountAssignment.DomainDns } };
                                            var AdId2 = await DataHistorySqlHelper.GetScalarResultAsync<Guid?>(DBConn, QueryGetAdAccount, Params, requestInfo, Token);
                                            if (AdId2 != null)
                                            {
                                                AdId = (Guid)AdId2;
                                                break;
                                            }
                                        }

                                        if (AdId == Guid.Empty)
                                            continue;

                                        var Params2 = new Dictionary<string, object>()
                                    {
                                        { "id", AdId },
                                        { "id_agent", id },
                                        { "account", name },
                                        { "sid", strSid },
                                        { "sid_bin", objSid },
                                        { "lastseen", lastAgent}
                                    };

                                        var n = await DataHistorySqlHelper.ExecuteAsync(DBConn, QueryInsertLocalAccount, Params2, requestInfo, Token);
                                        if (n > 0)
                                            Updated++;

                                        var Params3 = new Dictionary<string, object>()
                                    {
                                        { "id", AdId },
                                        { "lastAgent", lastAgent}
                                    };

                                        n = await DataHistorySqlHelper.ExecuteAsync(DBConn, QueryUpdateMainUserLastSeen, Params3, requestInfo, Token);

                                        continue;
                                    }

                                    // check if we have an azure ad account
                                    var azureId = cDataHistoryCollectorActiveDirectory.ConvertAzureSidToGuid(strSid);
                                    if (azureId != Guid.Empty)
                                    {
                                        try
                                        {

                                            var SqlParams3 = new Dictionary<string, object>()
                                            {
                                                { "id_agent" , id },
                                                { "id_azure", azureId },
                                                { "lastAgent", lastAgent }
                                            };

                                            var ret2 = await DataHistorySqlHelper.ExecuteAsync(DBConn, QueryInsertAzure, SqlParams3, requestInfo, Token);
                                            if (ret2 > 0)
                                                Updated++;

                                        }
                                        catch (Exception E)
                                        {
                                            LogException(E);
                                        }

                                        continue;
                                    }

                                    // process the normal ad account
                                    var SqlParams4 = new Dictionary<string, object>()
                                    {
                                        { "id_agent" , id },
                                        { "sid_bin", objSid },
                                        { "lastAgent", lastAgent }
                                    };
                                    var ret = await DataHistorySqlHelper.ExecuteAsync(DBConn, QueryInsert, SqlParams4, requestInfo, Token);


                                    if (ret >= 0)
                                        Count++;
                                    if (ret > 0)
                                        Updated++;
                                }
                                catch (Exception E)
                                {
                                    LogException(E);
                                }

                                _startTime2 = DateTime.UtcNow;
                            }
                            _duration += (DateTime.UtcNow - _startTime2).TotalMilliseconds;

                            Collector.DoProcessUiMessage(2, $"{Count} account processed, {Updated} inserted or updated.");

                            return true;
                        }

                    }
                }
                catch (Exception E)
                {
                    apiError = E.HResult;
                    LogException(E);
                }
                finally
                {
                    if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(QueryGet.Name, _startTime, _duration, SqlParams, apiError, requestInfo?.requestName);
                    if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(QueryGet.Name + "(total)", _startTime, SqlParams, apiError, requestInfo?.requestName);
                }

                Collector.DoProcessUiMessage(2, $"{RecCount} devices found, {Count} entries processed, {Updated} inserted or updated.");

                return found;
            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (CM != null) LogMethodEnd(CM);
            }

            return false;

        }

        private async Task<bool> ScanAccountsAsync(cDbConnection DBConn, cDbConnection AgentConn, DateTime lastScanTime, cF4sdWebRequestInfo requestInfo, int LogDeep, CancellationToken Token)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            try
            {
                Collector.DoProcessUiMessage(1, "Start scanning the user account properties...");

                var _eventTiming = await getActivityEventTiming(Token, requestInfo, LogDeep + 1);
                var _scanTime = lastScanTime - TimeSpan.FromMilliseconds(_eventTiming.userScan + _eventTiming.userSend) - TimeSpan.FromHours(1);

                bool _retVal = true;

                if (useOverallEventTables)
                {
                    if (!(EventTablesInfo?.Count >= 1) || EventTablesInfo.First().Value.From > _scanTime)
                        _retVal &= await ScanDevicesDailyAsync(DBConn, AgentConn, null, _scanTime, requestInfo, LogDeep + 1, Token);
                }

                if (useDailyEventTables)
                {
                    // get the list of daily event tables
                    var _listTables = await getDailyEventTables(_scanTime, DateTime.MaxValue, requestInfo, LogDeep + 1, Token);
                    foreach (var _tableName in _listTables)
                        _retVal &= await ScanAccountsDailyAsync(DBConn, AgentConn, _tableName.UserActivity, _scanTime, requestInfo, LogDeep + 1, Token);
                }

                return _retVal;
            }
            catch (Exception E)
            {
                Collector.DoProcessUiMessage(1, "Error while scanning the user account properties!");
                LogException(E);
            }
            finally
            {
                Collector.DoProcessUiMessage(1, "Scanning the user account properties finished.");
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }

            return false;
        }

        public override async Task<List<object>> GetIds(cF4sdConnectorIds IdEntry, enumFasdInformationClass InfoClass, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            await Task.CompletedTask;
            return new List<object>() { IdEntry.agentId };
        }

        public override async Task<List<cF4SDHealthCardRawData.cHealthCardTable>> GetTableResultsVirtualAsync(List<cDataHistoryConfigTable> Tables, Dictionary<enumFasdInformationClass, cF4sdConnectorIds> Identities, DateTime RefTime, int MaxAge, bool instantly, Guid? CacheId, CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            try
            {
                int ComputerId = -1;
                int? UserId = null;
                if (Identities.TryGetValue(enumFasdInformationClass.Computer, out var ComputerIdentity))
                    ComputerId = ComputerIdentity?.agentId ?? -1;
                if (Identities.TryGetValue(enumFasdInformationClass.User, out var UserIdentity))
                    UserId = UserIdentity?.agentId;

                var NeedJsons = false;
                var NeedMeasureCodes = false;
                foreach (var Table in Tables)
                {
                    if (Table.Type != eDataHistoryTableType.Static)
                        NeedMeasureCodes = true;
                    foreach (var Column in Table.Columns.Values)
                    {
                        if (!(Column is cDataHistoryConfigColumn col))
                            continue;
                        if (col.SourceJsonField != null)
                            NeedJsons = true;
                    }
                }

                Dictionary<string, cMeasureCodeInfo> MeasureCodes = null;
                Dictionary<string, string[]> JsonSettings = null;

                if (NeedMeasureCodes)
                    MeasureCodes = await getMeasureCodes(Token, requestInfo, LogDeep + 1).ConfigureAwait(false);

                if (NeedJsons)
                    JsonSettings = await getTypeJsonSettingsCodes(Token, requestInfo, LogDeep + 1).ConfigureAwait(false);

                var RetVal = new List<cF4SDHealthCardRawData.cHealthCardTable>();
                var lstTasks = new List<Task<List<cF4SDHealthCardRawData.cHealthCardTable>>>(Tables.Count);
                foreach (var Table in Tables)
                {
                    if (!instantly && CacheId != null && !Table.HasLateDelivery)
                        continue;

                    switch (Table.Name)
                    {
                        case constAgentUserTableName:
                            if (UserId == null || UserId == -1)
                                return null;
                            RetVal.Add(getUserInfo(UserIdentity));
                            break;
                        case constAgentDeviceTableName:
                            if (ComputerId == -1)
                                return null;
                            lstTasks.Add(getDeviceInfo(ComputerId, Token, requestInfo, LogDeep + 1)); ;
                            break;
                        case constAgentEventNumericTableName:
                        case constAgentEventStringTableName:
                            if (ComputerId == -1)
                                return null;
                            lstTasks.Add(getHistoricEventData(Table, ComputerId, MeasureCodes, JsonSettings, RefTime, MaxAge, instantly, CacheId != null, Token, requestInfo, LogDeep + 1));
                            break;
                    }
                }

                if (lstTasks.Count > 0)
                {
                    var Result = await Task.WhenAll(lstTasks).ConfigureAwait(false);
                    if (Result != null)
                        foreach (var Entry in Result)
                            if (Entry != null)
                                RetVal.AddRange(Entry);
                }

                if (RetVal.Count > 0)
                    return RetVal;
            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }

            return null;
        }

        private cF4SDHealthCardRawData.cHealthCardTable getUserInfo(cF4sdConnectorIds userInfo)
        {
            try
            {
                if (!(userInfo?.agentId is int id))
                    return null;

                var RetVal = new cF4SDHealthCardRawData.cHealthCardTable()
                {
                    Name = constAgentUserTableName,
                    InformationClass = enumFasdInformationClass.User,
                    Origin = enumDataHistoryOrigin.F4sdAgent,
                    IsStatic = true,
                    TableType = eDataHistoryTableType.Static
                };
                RetVal.Columns.Add("id", new cF4SDHealthCardRawData.cHealthCardTableColumn(RetVal) { ColumnName = "id", Values = new List<object>() { id } });
                RetVal.Columns.Add("accountType", new cF4SDHealthCardRawData.cHealthCardTableColumn(RetVal) { ColumnName = "accountType", Values = new List<object>() { userInfo.accountType.ToString() } });

                return RetVal;
            }
            catch (Exception E)
            {
                LogException(E);
            }

            return null;
        }

        private async Task<List<cF4SDHealthCardRawData.cHealthCardTable>> getDeviceInfo(int? agentId, CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            if (!(agentId is int id))
                return null;

            if (!DataHistorySqlHelper.GetWellKnownSqlStatement("GetComputerAttributesFromAgent", out var Query))
                return null;

            var Params = new Dictionary<string, object>() { { "id", id } };

            try
            {

                if (!Collector.ClusterConfig.Tables.TryGetValue(constAgentDeviceTableName, out var Table))
                    return null;

                var AgentDb = Collector.InfrastructureConfig.ClientAgent.Connection;


                using (var ConnAgent = new cDbConnection(AgentDb))
                {
                    if (!ConnAgent.IsOpen)
                    {
                        LogEntry($"Could not open client agent database '{AgentDb.Database}', aborting  query'", LogLevels.Error);
                        return null;
                    }

                    using (var Reader = await DataHistorySqlHelper.GetDataReaderAsync(ConnAgent, Query, Params, Token))
                    {
                        if (Reader != null)
                            try
                            {
                                if (Reader.HasRows)
                                {
                                    var dicCols = Table.Columns.Values.Where(x => x.SourceName != null).ToDictionary<cDataHistoryConfigColumnBase, string>(x => x.SourceName);
                                    var RetVal = new cF4SDHealthCardRawData.cHealthCardTable()
                                    {
                                        InformationClass = enumFasdInformationClass.Computer,
                                        Origin = enumDataHistoryOrigin.F4sdAgent
                                    };
                                    RetVal.Columns.Add("id", new cF4SDHealthCardRawData.cHealthCardTableColumn(RetVal) { ColumnName = "id", Values = new List<object>() { id } });
                                    while (await Reader.ReadAsync())
                                    {
                                        try
                                        {
                                            var Name = Reader.GetString(0);
                                            var JsonValue = Reader.GetString(1);
                                            if (dicCols.TryGetValue(Name, out var Column))
                                            {
                                                var objValue = cDataHistoryCollector.getJsonValueFromString(JsonValue, Column.ValueType);
                                                RetVal.Columns.Add(Column.Name, new cF4SDHealthCardRawData.cHealthCardTableColumn(RetVal) { ColumnName = Column.Name, Values = new List<object>() { objValue } });
                                            }
                                        }
                                        catch (Exception E)
                                        {
                                            LogException(E);
                                        }
                                    }

                                    RetVal.Name = Table.Name;
                                    RetVal.InformationClass = Table.ParentCluster.InformationClass;
                                    RetVal.IsStatic = true;
                                    RetVal.TableType = eDataHistoryTableType.Static;

                                    if (dicCols.ContainsKey("DomainNB") && RetVal.Columns.TryGetValue("DomainDns", out var _col))
                                    {
                                        var domainDns = _col.Values.First().ToString();
                                        if (!string.IsNullOrEmpty(domainDns))
                                        {

                                        }
                                    }

                                    return new List<cF4SDHealthCardRawData.cHealthCardTable>() { RetVal };
                                }
                            }
                            catch (Exception E)
                            {
                                LogException(E);
                            }

                    }


                }
            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(Query.Name, _startTime, Params, 0, requestInfo?.requestName);
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }

            return null;
        }

        private class cMeasureCodeInfo
        {
            internal int MeasureCode;
            internal int TypeCode;
            internal int StatusCode;
        }

        private async Task<Dictionary<string, cMeasureCodeInfo>> getMeasureCodes(CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            if (!DataHistorySqlHelper.GetWellKnownSqlStatement("GetMeasureCodesFromAgent", out var Query))
                return null;

            var _params = new Dictionary<string, object>() { { "ORGCODE", OrganizationCode } };

            var sqlerror = 0;
            var sqlStartTime = DateTime.UtcNow;

            try
            {

                var AgentDb = Collector.InfrastructureConfig.ClientAgent.Connection;

                using (var ConnAgent = new cDbConnection(AgentDb))
                {
                    if (!ConnAgent.IsOpen)
                    {
                        LogEntry($"Could not open client agent database '{AgentDb.Database}', aborting  query'", LogLevels.Error);
                        return null;
                    }
                    using (var Reader = await DataHistorySqlHelper.GetDataReaderAsync(ConnAgent, Query, _params, Token))
                    {
                        if (Reader != null)
                            try
                            {
                                if (Reader.HasRows)
                                {
                                    var RetVal = new Dictionary<string, cMeasureCodeInfo>();
                                    while (await Reader.ReadAsync())
                                    {
                                        try
                                        {
                                            var Id = Reader.GetInt32(0);
                                            var Name = Reader.GetString(1);
                                            var TypeCode = Reader.GetInt32(2);
                                            var StatusCode = Reader.IsDBNull(3) ? 0 : Reader.GetInt32(3);

                                            var Info = new cMeasureCodeInfo() { MeasureCode = Id, TypeCode = TypeCode, StatusCode = StatusCode };
                                            if (RetVal.TryGetValue(Name, out var _mc))
                                            {
                                                if (StatusCode < _mc.StatusCode)
                                                    RetVal[Name] = Info;
                                            }
                                            else
                                                RetVal.Add(Name, Info);
                                        }
                                        catch (Exception E)
                                        {
                                            LogException(E);
                                        }
                                    }

                                    return RetVal;
                                }
                            }
                            catch (Exception E)
                            {
                                LogException(E);
                            }
                    }
                }

            }
            catch (Exception E)
            {
                sqlerror = E.HResult;
                LogException(E);
            }
            finally
            {
                if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(Query.Name, sqlStartTime, _params, sqlerror, requestInfo?.requestName);
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }

            return null;
        }

        private async Task<Dictionary<string, string[]>> getTypeJsonSettingsCodes(CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            try
            {
                if (!DataHistorySqlHelper.GetWellKnownSqlStatement("GetTypeJsonSettingsFromAgent", out var Query))
                    return null;

                var AgentDb = Collector.InfrastructureConfig.ClientAgent.Connection;

                var sqlerror = 0;
                var sqlStartTime = DateTime.UtcNow;

                try
                {
                    using (var ConnAgent = new cDbConnection(AgentDb))
                    {
                        if (!ConnAgent.IsOpen)
                        {
                            LogEntry($"Could not open client agent database '{AgentDb.Database}', aborting  query'", LogLevels.Error);
                            return null;
                        }

                        using (var Reader = await DataHistorySqlHelper.GetDataReaderAsync(ConnAgent, Query, null, Token))
                        {
                            if (Reader != null)
                                try
                                {
                                    if (Reader.HasRows)
                                    {
                                        var RetVal = new Dictionary<string, string[]>();
                                        while (await Reader.ReadAsync())
                                        {
                                            try
                                            {
                                                var Name = Reader.GetString(0);
                                                var strJson = Reader.GetString(1);
                                                var objJson = JsonConvert.DeserializeObject<jsonTypeSettings>(strJson);
                                                if (objJson?.values != null)
                                                    RetVal[Name] = objJson.values;
                                            }
                                            catch (Exception E)
                                            {
                                                LogException(E);
                                            }
                                        }

                                        return RetVal;
                                    }
                                }
                                catch (Exception E)
                                {
                                    LogException(E);
                                }
                        }
                    }
                }
                catch (Exception E)
                {
                    sqlerror = E.HResult;
                    LogException(E);
                }
                finally
                {
                    if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(Query.Name, sqlStartTime, "", sqlerror, requestInfo?.requestName);
                }

            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }

            return null;
        }

        private async Task<cActivityEventTiming> getActivityEventTiming(CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }

            if (!DataHistorySqlHelper.GetWellKnownSqlStatement("GetActivityPeriods", out var Query))
                return null;

            var Params = new Dictionary<string, object>() { { "ORGCODE", OrganizationCode } };

            var retVal = new cActivityEventTiming();
            var _startTime = DateTime.UtcNow;
            try
            {
                var AgentDb = Collector.InfrastructureConfig.ClientAgent.Connection;
                using (var ConnAgent = new cDbConnection(AgentDb))
                {
                    if (!ConnAgent.IsOpen)
                    {
                        LogEntry($"Could not open client agent database '{AgentDb.Database}', aborting  query'", LogLevels.Error);
                        return null;
                    }

                    using (var Reader = await DataHistorySqlHelper.GetDataReaderAsync(ConnAgent, Query, Params, Token))
                    {
                        if (Reader != null)
                            try
                            {
                                if (Reader.HasRows)
                                {
                                    while (await Reader.ReadAsync())
                                    {
                                        try
                                        {
                                            var _mc = Reader.GetInt32(0);
                                            var _tc = Reader.GetInt32(1);
                                            var _oc = Reader.IsDBNull(2) ? (int?)null : Reader.GetInt32(2);
                                            var _val = int.Parse(Reader.GetString(3));

                                            switch (_mc)
                                            {
                                                case 40:
                                                    switch (_tc)
                                                    {
                                                        case 1:
                                                            if (_oc != null || retVal.deviceSend == 0)
                                                                retVal.deviceSend = _val;
                                                            break;
                                                        case 2:
                                                            if (_oc != null || retVal.deviceScan == 0)
                                                                retVal.deviceScan = _val;
                                                            break;
                                                    }
                                                    break;
                                                case 41:
                                                    switch (_tc)
                                                    {
                                                        case 1:
                                                            if (_oc != null || retVal.userSend == 0)
                                                                retVal.userSend = _val;
                                                            break;
                                                        case 2:
                                                            if (_oc != null || retVal.userScan == 0)
                                                                retVal.userScan = _val;
                                                            break;
                                                    }
                                                    break;
                                            }

                                        }
                                        catch (Exception E)
                                        {
                                            LogException(E);
                                        }
                                    }
                                }
                            }
                            catch (Exception E)
                            {
                                LogException(E);
                            }

                    }
                }
            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(Query.Name, _startTime, Params, 0, requestInfo?.requestName);
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }

            return retVal;
        }

        private static object GetValueFromDbField(object val, bool needJsonDecode)
        {
            try
            {
                if (val == DBNull.Value)
                    val = null;
                if (val is string strValue)
                {
                    if (strValue.StartsWith("\""))
                        val = JsonConvert.DeserializeObject<string>(strValue);
                    else if (needJsonDecode && strValue.StartsWith("["))
                        val = JsonConvert.DeserializeObject<object[]>(strValue);
                    else if (needJsonDecode &&strValue.StartsWith("{"))
                        val = JsonConvert.DeserializeObject<cDataHistoryCustomJsonMeasure>(strValue);
                    else
                        val = strValue;
                }
            }
            catch (Exception E)
            {
                LogException(E, LogLevels.Warning);
            }

            return val;
        }

        public override bool CheckIfLateDelivery(cDataHistoryConfigTable Table)
        {
            return Table.HasLateDelivery;
        }

        private async Task getHistoricEventDataTotal(int AgentId, string TableType, string mcFilter, Dictionary<int, cMeasureCodeInformation> McColumns, int MaxAge, DateTime RefTime, CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            try
            {
                string QueryName = $"GetComputerHistoricEventsTotal{TableType}FromAgent";
                if (!DataHistorySqlHelper.GetWellKnownSqlStatement(QueryName, out var Query))
                    return;

                var strQuery = string.Format(Query.Query, mcFilter);
                var Params = new Dictionary<string, object>() { { "ID", AgentId }, { "Age", MaxAge } };

                var sqlerror = 0;
                var sqlStartTime = DateTime.UtcNow;

                try
                {
                    var AgentDb = Collector.InfrastructureConfig.ClientAgent.Connection;

                    using (var ConnAgent = new cDbConnection(AgentDb))
                    {
                        if (!ConnAgent.IsOpen)
                        {
                            LogEntry($"Could not open client agent database '{AgentDb.Database}', aborting  query'", LogLevels.Error);
                            return;
                        }

                        using (var Reader = await DataHistorySqlHelper.GetDataReaderAsync(ConnAgent.Connection, strQuery, Params, Token))
                        {
                            if (Reader != null)
                                try
                                {
                                    if (Reader.HasRows)
                                    {
                                        while (await Reader.ReadAsync())
                                        {
                                            var _MC = Reader.GetInt32(0);
                                            var _EventTime = Reader.GetDateTime(1);
                                            var _EventDuration = Reader.IsDBNull(3) ? 1 : Reader.GetInt32(3);
                                            if (McColumns.TryGetValue(_MC, out var aggEntry))
                                            {
                                                object Value = GetValueFromDbField(Reader.GetValue(2), aggEntry.needsJson);
                                                foreach (var aggValEntry in aggEntry.Entries)
                                                    aggValEntry.addValue(Value, _EventTime, RefTime, _EventDuration);
                                            }
                                        }
                                    }
                                }
                                catch (Exception E)
                                {
                                    LogException(E);
                                }
                        }
                    }
                }
                catch (Exception E)
                {
                    sqlerror = E.HResult;
                    LogException(E);
                }
                finally
                {
                    Params.Add("Filter", mcFilter);
                    if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(Query.Name, sqlStartTime, Params, sqlerror, requestInfo?.requestName);
                }
            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }
        }

        private async Task getHistoricEventDataDaily(int AgentId, string TableName, string mcFilter, Dictionary<int, cMeasureCodeInformation> McColumns, int MaxAge, DateTime RefTime, CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            try
            {
                if (!DataHistorySqlHelper.GetWellKnownSqlStatement("GetComputerHistoricEventsDailyFromAgent", out var Query))
                    return;

                var strQuery = string.Format(Query.Query, TableName, mcFilter);
                var Params = new Dictionary<string, object>() { { "ID", AgentId } };

                var sqlerror = 0;
                var sqlStartTime = DateTime.UtcNow;

                try
                {
                    var AgentDb = Collector.InfrastructureConfig.ClientAgent.Connection;

                    using (var ConnAgent = new cDbConnection(AgentDb))
                    {
                        if (!ConnAgent.IsOpen)
                        {
                            LogEntry($"Could not open client agent database '{AgentDb.Database}', aborting  query'", LogLevels.Error);
                            return;
                        }

                        using (var Reader = await DataHistorySqlHelper.GetDataReaderAsync(ConnAgent.Connection, strQuery, Params, Token))
                        {
                            if (Reader != null)
                                try
                                {
                                    if (Reader.HasRows)
                                    {
                                        while (await Reader.ReadAsync())
                                        {
                                            var _MC = Reader.GetInt32(0);
                                            if (_MC == 24)
                                            {
                                            }
                                            var _EventTime = Reader.GetDateTime(1);
                                            if (_EventTime < RefTime - TimeSpan.FromDays(MaxAge))
                                                continue;

                                            var _EventDuration = Reader.IsDBNull(3) ? 1 : Reader.GetInt32(3);

                                            if (McColumns.TryGetValue(_MC, out var aggEntry))
                                            {
                                                object Value = GetValueFromDbField(Reader.GetValue(2), aggEntry.needsJson);
                                                foreach (var aggValEntry in aggEntry.Entries)
                                                    aggValEntry.addValue(Value, _EventTime, RefTime, _EventDuration);
                                            }
                                        }
                                    }
                                }
                                catch (Exception E)
                                {
                                    LogException(E);
                                }
                        }
                    }
                }
                catch (Exception E)
                {
                    sqlerror = E.HResult;
                    LogException(E);
                }
                finally
                {
                    Params.Add("TableName", TableName);
                    Params.Add("Filter", mcFilter);
                    if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(Query.Name, sqlStartTime, Params, sqlerror, requestInfo?.requestName);
                }
            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }
        }


        private async Task getHistoricEventDataDailyUsage(int AgentId, string TableName, cMeasureCodeInformation aggEntry, int MaxAge, DateTime RefTime, CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            try
            {
                if (!DataHistorySqlHelper.GetWellKnownSqlStatement("GetComputerHistoricEventsDailyUsageFromAgent", out var Query))
                    return;

                var strQuery = string.Format(Query.Query, TableName);
                var Params = new Dictionary<string, object>() { { "ID", AgentId } };

                var sqlerror = 0;
                var sqlStartTime = DateTime.UtcNow;

                try
                {
                    var AgentDb = Collector.InfrastructureConfig.ClientAgent.Connection;

                    using (var ConnAgent = new cDbConnection(AgentDb))
                    {
                        if (!ConnAgent.IsOpen)
                        {
                            LogEntry($"Could not open client agent database '{AgentDb.Database}', aborting  query'", LogLevels.Error);
                            return;
                        }

                        using (var Reader = await DataHistorySqlHelper.GetDataReaderAsync(ConnAgent.Connection, strQuery, Params, Token))
                        {
                            if (Reader != null)
                                try
                                {
                                    if (Reader.HasRows)
                                    {
                                        while (await Reader.ReadAsync())
                                        {
                                            var _MC = Reader.GetInt32(0);
                                            var _EventTime = Reader.GetDateTime(1);
                                            if (_EventTime < RefTime - TimeSpan.FromDays(MaxAge))
                                                continue;

                                            var _EventDuration = Reader.IsDBNull(3) ? 1 : Reader.GetInt32(3);
                                            object Value = GetValueFromDbField(Reader.GetValue(2), aggEntry.needsJson);
                                            foreach (var aggValEntry in aggEntry.Entries)
                                                aggValEntry.addValue(Value, _EventTime, RefTime, _EventDuration);
                                        }
                                    }
                                }
                                catch (Exception E)
                                {
                                    LogException(E);
                                }
                        }
                    }
                }
                catch (Exception E)
                {
                    sqlerror = E.HResult;
                    LogException(E);
                }
                finally
                {
                    Params.Add("TableName", TableName);
                    if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(Query.Name, sqlStartTime, Params, sqlerror, requestInfo?.requestName);
                }
            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }
        }

        private async Task<List<cF4SDHealthCardRawData.cHealthCardTable>> getHistoricEventData(cDataHistoryConfigTable Table, int AgentId, Dictionary<string, cMeasureCodeInfo> MeasureCodes, Dictionary<string, string[]> JsonSettings, DateTime RefTime, int MaxAge, bool instantly, bool IsSecondCall, CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug)
            {
                CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM);
                LogEntry($"Table: {Table.Name}, HasLateDelivery: {Table.HasLateDelivery}, Instantly: {instantly}, IsSecondCall: {IsSecondCall}", LogLevels.Debug);
            }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            string TableType = null;
            switch (Table.Name)
            {
                case constAgentEventNumericTableName:
                    TableType = "Numeric";
                    break;
                case constAgentEventStringTableName:
                    TableType = "String";
                    break;
                default:
                    return null;
            }

            try
            {
                var McColumns = new Dictionary<int, cMeasureCodeInformation>();
                foreach (var _Entry in Table.Columns.Values)
                {
                    if (!(_Entry is cDataHistoryConfigColumn col))
                        continue;

                    if (!MeasureCodes.TryGetValue(col.SourceName, out var _mc))
                        continue;

                    if (!instantly)
                    {
                        if (IsSecondCall)
                        {
                            if (!_Entry.LateDelivery)
                                continue;
                        }
                        else
                        {
                            if (_Entry.LateDelivery)
                            {
                                continue;
                            }
                        }
                    }

                    if (!McColumns.TryGetValue(_mc.MeasureCode, out var _colList))
                    {
                        _colList = new cMeasureCodeInformation();
                        McColumns[_mc.MeasureCode] = _colList;
                    }
                    var AE = new cHistoricAggregationEntry(col, MaxAge + 1, JsonSettings);
                    _colList.AddEntry(AE);
                }

                var mcFilter = "";
                var mcFilter2 = "";         // filter without activity measure codes (40,41)
                foreach (var _mc in McColumns.Keys)
                {
                    if (mcFilter != "")
                        mcFilter += ",";
                    mcFilter += _mc.ToString();

                    if (_mc != 40 && _mc != 41)
                    {
                        if (mcFilter2 != "")
                            mcFilter2 += ",";
                        mcFilter2 += _mc.ToString();
                    }
                }


                var _listTask = new List<Task>();

                if (useOverallEventTables)
                {
                    if (!(EventTablesInfo?.Count >= 1) || EventTablesInfo.First().Value.From > DateTime.UtcNow - TimeSpan.FromDays(MaxAge))
                        _listTask.Add(getHistoricEventDataTotal(AgentId, TableType, mcFilter, McColumns, MaxAge, RefTime, Token, requestInfo, LogDeep + 1));
                }

                if (useDailyEventTables)
                {
                    var _tableNames = await getDailyEventTables(MaxAge, requestInfo, LogDeep + 1, Token);
                    if (_tableNames != null)
                    {
                        foreach (var tableNameInfo in _tableNames)
                        {
                            string _tableName = null;
                            switch (TableType)
                            {
                                case "Numeric":
                                    if (McColumns.TryGetValue(40, out var aggEntry1))
                                        _listTask.Add(getHistoricEventDataDailyUsage(AgentId, tableNameInfo.DeviceActivity, aggEntry1, MaxAge, RefTime, Token, requestInfo, LogDeep + 1));
                                    if (McColumns.TryGetValue(41, out var aggEntry2))
                                        _listTask.Add(getHistoricEventDataDailyUsage(AgentId, tableNameInfo.UserActivity, aggEntry2, MaxAge, RefTime, Token, requestInfo, LogDeep + 1));
                                    _tableName = tableNameInfo.SimpleValues;
                                    break;
                                case "String":
                                    _tableName = tableNameInfo.JsonData;
                                    break;
                            }

                            if (_tableName != null)
                            {
                                _listTask.Add(getHistoricEventDataDaily(AgentId, _tableName, mcFilter, McColumns, MaxAge, RefTime, Token, requestInfo, LogDeep + 1));
                            }
                        }
                    }
                }

                await Task.WhenAll(_listTask).ConfigureAwait(false);

                var RetValLatest = new cF4SDHealthCardRawData.cHealthCardTable()
                {
                    Name = Table.Name + "-latest",
                    InformationClass = Table.ParentCluster.InformationClass,
                    Origin = Table.ParentCluster.Origin,
                    IsStatic = true,
                    TableType = eDataHistoryTableType.Static
                };

                var RetValHistoric = new cF4SDHealthCardRawData.cHealthCardTable()
                {
                    Name = Table.Name,
                    InformationClass = Table.ParentCluster.InformationClass,
                    Origin = Table.ParentCluster.Origin,
                    IsStatic = false,
                    TableType = eDataHistoryTableType.History,
                    AlternateStaticTable = RetValLatest.Name
                };

                foreach (var Entry in McColumns)
                {
                    if (Entry.Key == 29)
                    {
                    }
                    foreach (var subEntry in Entry.Value.Entries)
                    {
                        var col = new cF4SDHealthCardRawData.cHealthCardTableColumn(RetValHistoric)
                        {
                            ColumnName = subEntry.Column.Name,
                            Values = subEntry.getValues()
                        };
                        RetValHistoric.Columns[subEntry.Column.Name] = col;

                        object Value2 = null;
                        if (subEntry.Column.AggregationType == eDataHistoryAggregationType.latestTime)
                            Value2 = subEntry.LastestEvent;
                        else
                            Value2 = subEntry.LatestValue;

                        var col2 = new cF4SDHealthCardRawData.cHealthCardTableColumn(RetValLatest)
                        {
                            ColumnName = subEntry.Column.Name,
                            Values = new List<object> { Value2 }
                        };
                        RetValLatest.Columns[subEntry.Column.Name] = col2;
                    }
                }

                if (!instantly && !IsSecondCall && Table.HasLateDelivery)
                {
                    foreach (var _col in Table.Columns.Values)
                    {
                        if (_col.LateDelivery)
                        {
                            RetValHistoric.Columns.Add(_col.Name, new cF4SDHealthCardRawData.cHealthCardTableColumn(RetValHistoric)
                            {
                                ColumnName = _col.Name,
                                IsIncomplete = true
                            });
                            RetValLatest.Columns.Add(_col.Name, new cF4SDHealthCardRawData.cHealthCardTableColumn(RetValLatest)
                            {
                                ColumnName = _col.Name,
                                IsIncomplete = true
                            });
                        }
                    }
                    RetValHistoric.IsIncomplete = true;
                    RetValLatest.IsIncomplete = true;
                }

                return new List<cF4SDHealthCardRawData.cHealthCardTable>() { RetValHistoric, RetValLatest };

            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }
            return null;
        }

        public override async Task<List<cF4SDHealthCardRawData.cHealthCardDetailsTable>> GetDetailsTableResultsVirtualAsync(List<cDataHistoryConfigTable> Tables, Dictionary<enumFasdInformationClass, cF4sdConnectorIds> Identities, DateTime RefTime, int MaxAge, CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            try
            {
                int? ComputerId = null;
                int? UserId = null;
                if (Identities.TryGetValue(enumFasdInformationClass.Computer, out var ComputerIdentity))
                    ComputerId = ComputerIdentity?.agentId;
                if (Identities.TryGetValue(enumFasdInformationClass.User, out var UserIdentity))
                    UserId = UserIdentity?.agentId;

                var NeedJsons = false;
                var NeedMeasureCodes = false;
                foreach (var Table in Tables)
                {
                    if (Table.Type != eDataHistoryTableType.Static)
                        NeedMeasureCodes = true;
                    foreach (var Column in Table.Columns.Values)
                    {
                        if (!(Column is cDataHistoryConfigColumn col))
                            continue;
                        if (col.SourceJsonField != null)
                            NeedJsons = true;
                    }
                }

                Dictionary<string, cMeasureCodeInfo> MeasureCodes = null;
                Dictionary<string, string[]> JsonSettings = null;

                if (NeedMeasureCodes)
                    MeasureCodes = await getMeasureCodes(Token, requestInfo, LogDeep + 1).ConfigureAwait(false);

                if (NeedJsons)
                    JsonSettings = await getTypeJsonSettingsCodes(Token, requestInfo, LogDeep + 1).ConfigureAwait(false);

                var RetVal = new List<cF4SDHealthCardRawData.cHealthCardDetailsTable>();
                var lstTasks = new List<Task<cF4SDHealthCardRawData.cHealthCardDetailsTable>>(Tables.Count);
                foreach (var Table in Tables)
                {
                    lstTasks.Add(getHistoricEventDetails(Table, ComputerId, MeasureCodes, JsonSettings, RefTime, MaxAge, Token, requestInfo, LogDeep + 1));
                }

                if (lstTasks.Count > 0)
                {
                    var Result = await Task.WhenAll(lstTasks).ConfigureAwait(false);
                    if (Result != null)
                        foreach (var Entry in Result)
                            if (Entry != null)
                                RetVal.Add(Entry);
                }

                if (RetVal.Count > 0)
                    return RetVal;
            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }

            return null;
        }

        internal class cEventDetailsColumnInfo
        {
            internal int index = -1;
            internal int JsonIndex = -1;
            internal cDataHistoryConfigColumn column = null;
        }

        private object ConvertDbObjectToF4sdType(object Value, enumFasdValueType valueType)
        {
            try
            {
                if (Value is null || Value is DBNull)
                    return null;

                switch (valueType)
                {
                    case enumFasdValueType.INT:
                        if (Value is Int32)
                            return Value;
                        if (Value is Int64 _i64)
                            if (_i64 >= Int32.MinValue && _i64 <= Int32.MinValue)
                                return Convert.ToInt32(Value);
                            else
                                return null;
                        if (Int32.TryParse(Value.ToString(), out var _r1))
                            return _r1;
                        return null;
                    case enumFasdValueType.BIGINT:
                        if (Value is Int64)
                            return Value;
                        if (Value is Int32 _i32)
                            return (Int64)_i32;
                        if (Int64.TryParse(Value.ToString(), out var _r2))
                            return _r2;
                        return null;
                    case enumFasdValueType.FLOAT:
                        if (Value is Int64 || Value is Int32 || Value is float || Value is double || Value is Single)
                            return Convert.ToDouble(Value);
                        if (Double.TryParse(Value.ToString(), out var _r3))
                            return _r3;
                        return null;
                    case enumFasdValueType.DATETIME:
                        if (Value is DateTime)
                            return Value;
                        if (DateTime.TryParse(Value.ToString(), out var _r4))
                            return _r4;
                        return null;
                    default:
                        return Value.ToString();
                }

            }
            catch (Exception E)
            {
                LogException(E);
            }

            return null;
        }

        internal class cHistoricEventRowInfos
        {
            internal List<cEventDetailsColumnInfo> rows = new List<cEventDetailsColumnInfo>();
            internal bool needJson = false;

            internal void AddEntry(cEventDetailsColumnInfo row)
            {
                rows.Add(row);
                if (row.JsonIndex >= 0)
                    needJson = true;
            }
        }

        internal class cHistoricEventDetailsInfo
        {
            internal cDataHistoryConfigTable Table;
            internal cF4SDHealthCardRawData.cHealthCardDetailsTable Result;
            internal SortedList<int, SortedDictionary<DateTime, object[]>> _values;
            internal List<cHistoricEventRowInfos> rows;
            internal int valueCount;
            internal string[] _jsonFields;
            internal cDataHistoryConfigColumnBase colTime;
            internal DateTime RefTime;
            internal int MaxAge;
        }

        private async Task<bool> getHistoricEventDetailsDaily(cHistoricEventDetailsInfo _info, cDbQuery Query, Dictionary<string, object> Params, CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            var sqlerror = 0;
            var sqlStartTime = DateTime.UtcNow;

            try
            {
                var AgentDb = Collector.InfrastructureConfig.ClientAgent.Connection;
                using (var ConnAgent = new cDbConnection(AgentDb))
                {
                    if (!ConnAgent.IsOpen)
                    {
                        LogEntry($"Could not open client agent database '{AgentDb.Database}', aborting  query'", LogLevels.Error);
                        return false;
                    }

                    using (var Reader = await DataHistorySqlHelper.GetDataReaderAsync(ConnAgent, Query, Params, Token))
                    {
                        if (Reader != null)
                            try
                            {
                                if (Reader.HasRows)
                                {
                                    lock (_info)
                                    {
                                        if (_info.rows.Count == 0)
                                        {
                                            for (int i = 0; i < Reader.FieldCount; i++)
                                            {
                                                cHistoricEventRowInfos _rows = null;
                                                var colName = Reader.GetName(i);
                                                if (!string.IsNullOrWhiteSpace(colName))
                                                {
                                                    switch (colName)
                                                    {
                                                        case "id":
                                                        case "event":
                                                            continue;
                                                    }

                                                    foreach (var Entry in _info.Table.Columns.Values)
                                                    {
                                                        if (!(Entry is cDataHistoryConfigColumn column))
                                                            continue;

                                                        if (column.SourceName == colName)
                                                        {
                                                            if (_rows == null)
                                                                _rows = new cHistoricEventRowInfos();
                                                            _info.Result.Columns.Add(column.Name);

                                                            var JsonIndex = -1;
                                                            if (column.SourceJsonField != null && _info._jsonFields != null)
                                                            {
                                                                for (int j = 0; j < _info._jsonFields.Length; j++)
                                                                    if (_info._jsonFields[j] == column.SourceJsonField)
                                                                    {
                                                                        JsonIndex = j;
                                                                        break;
                                                                    }
                                                            }

                                                            _rows.AddEntry(new cEventDetailsColumnInfo() { index = _info.Result.Columns.Count - 1, JsonIndex = JsonIndex, column = column });
                                                        }
                                                    }

                                                }

                                                _info.rows.Add(_rows);
                                            }

                                            _info.valueCount = _info.Result.Columns.Count;
                                            foreach (var Entry in _info.Table.Columns.Values)
                                            {
                                                if (!(Entry is cDataHistoryConfigColumnComputation column))
                                                    continue;
                                                _info.valueCount++;
                                            }

                                        }
                                    }

                                    while (await Reader.ReadAsync())
                                    {
                                        var arrVals = new object[Reader.FieldCount];
                                        if (Reader.GetValues(arrVals) == Reader.FieldCount)
                                        {
                                            var EventTime = DateTime.MinValue;
                                            var arrValues = new object[_info.valueCount];
                                            for (int i = 0; i < Reader.FieldCount; i++)
                                            {
                                                var listCols = _info.rows[i];
                                                if (listCols != null && listCols.rows.Count > 0)
                                                {
                                                    var val = GetValueFromDbField(arrVals[i], listCols.needJson);
                                                    object resVal = null;
                                                    foreach (var Entry in listCols.rows)
                                                    {
                                                        if (Entry.column == _info.colTime && (val is DateTime evt))
                                                            EventTime = evt;

                                                        if (val is object[] arrObj)
                                                        {
                                                            if (Entry.JsonIndex >= 0 && Entry.JsonIndex < arrObj.Length)
                                                                resVal = arrObj[Entry.JsonIndex];
                                                        }
                                                        else
                                                            resVal = val;

                                                        resVal = ConvertDbObjectToF4sdType(resVal, Entry.column.ValueType);
                                                        arrValues[Entry.index] = resVal;
                                                    }
                                                }
                                            }

                                            var indexDay = -cDataHistoryCollector.getDayIndex(EventTime, _info.RefTime, _info.MaxAge + 1);
                                            lock (_info._values)
                                            {
                                                if (!_info._values.TryGetValue(indexDay, out var entryDay))
                                                {
                                                    entryDay = new SortedDictionary<DateTime, object[]>(new DuplicateKeyComparer<DateTime>()) { { EventTime, arrValues } };
                                                    _info._values.Add(indexDay, entryDay);
                                                }
                                                else
                                                {
                                                    if (entryDay.Count < constMaxDetailsEventsPerDay)
                                                        entryDay.Add(EventTime, arrValues);
                                                }
                                            }
                                        }
                                    }


                                }
                            }
                            catch (Exception E)
                            {
                                LogException(E);
                            }
                    }
                }

                return true;
            }
            catch (Exception E)
            {
                sqlerror = E.HResult;
                LogException(E);
            }
            finally
            {
                if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(Query.Name, sqlStartTime, Params, sqlerror, requestInfo?.requestName);
            }

            return false;
        }

        private async Task<cF4SDHealthCardRawData.cHealthCardDetailsTable> getHistoricEventDetails(cDataHistoryConfigTable Table, int? AgentId, Dictionary<string, cMeasureCodeInfo> MeasureCodes, Dictionary<string, string[]> JsonSettings, DateTime RefTime, int MaxAge, CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            try
            {

                if (!Table.Columns.TryGetValue("event", out var colEvent))
                {
                    LogEntry($"Could not find the event column in table '{Table.Name}'", LogLevels.Warning);
                    return null;
                }

                if (!Table.Columns.TryGetValue("time", out var colTime))
                {
                    LogEntry($"Could not find the time column in table '{Table.Name}'", LogLevels.Warning);
                    return null;
                }

                if (!MeasureCodes.TryGetValue(colEvent.SourceName, out var _mc))
                {
                    LogEntry($"Could not find the measure code for '{colEvent.SourceName}' column in table '{Table.Name}'", LogLevels.Warning);
                    return null;
                }

                string[] _jsonFields = null;
                if (JsonSettings != null && !JsonSettings.TryGetValue(colEvent.SourceName, out _jsonFields))
                    _jsonFields = null;

                var _info = new cHistoricEventDetailsInfo()
                {
                    Table = Table,
                    Result = new cF4SDHealthCardRawData.cHealthCardDetailsTable
                    {
                        Columns = new List<string>(),
                        Name = Table.Name,
                        Values = new Dictionary<int, List<object[]>>()
                    },
                    _values = new SortedList<int, SortedDictionary<DateTime, object[]>>(),
                    rows = new List<cHistoricEventRowInfos>(),
                    valueCount = 0,
                    _jsonFields = _jsonFields,
                    colTime = colTime,
                    RefTime = RefTime,
                    MaxAge = MaxAge
                };

                var _listTask = new List<Task>();

                if (useOverallEventTables)
                {
                    if (!(EventTablesInfo?.Count >= 1) || EventTablesInfo.First().Value.From > DateTime.UtcNow - TimeSpan.FromDays(MaxAge))
                    {
                        var Params = new Dictionary<string, object>() { { "ID", AgentId }, { "MC", _mc.MeasureCode }, { "MINDATE", DateTime.UtcNow - TimeSpan.FromDays(MaxAge) } };

                        var strQuery = "GetDetailEventValuesFromAgent";
                        if (_mc.TypeCode == 1)
                            strQuery = "GetDetailEventValuesNumericFromAgent";

                        if (DataHistorySqlHelper.GetWellKnownSqlStatement(strQuery, out var Query))
                            _listTask.Add(getHistoricEventDetailsDaily(_info, Query, Params, Token, requestInfo, LogDeep + 1));

                    }
                }

                if (useDailyEventTables)
                {
                    if (DataHistorySqlHelper.GetWellKnownSqlStatement("GetDetailEventValuesDailyFromAgent", out var Query))
                    {
                        var _tableNames = await getDailyEventTables(MaxAge, requestInfo, LogDeep + 1, Token);
                        if (_tableNames != null)
                        {
                            foreach (var tableNameInfo in _tableNames)
                            {
                                string _tableName = null;
                                if (_mc.TypeCode == 1)
                                    _tableName = tableNameInfo.SimpleValues;
                                else
                                    _tableName = tableNameInfo.JsonData;

                                var Query2 = Query.Clone(string.Format(Query.Query, _tableName));
                                var Params = new Dictionary<string, object>() { { "ID", AgentId }, { "MC", _mc.MeasureCode }, { "MINDATE", DateTime.UtcNow - TimeSpan.FromDays(MaxAge) } };
                                _listTask.Add(getHistoricEventDetailsDaily(_info, Query2, Params, Token, requestInfo, LogDeep + 1));
                            }
                        }
                    }
                }

                await Task.WhenAll(_listTask).ConfigureAwait(false);

                foreach (var _entry in _info._values)
                    _info.Result.Values.Add(-_entry.Key, _entry.Value.Values.ToList());


                cDataHistoryCollector.ComputeEventDetailColumns(Table, _info.Result);

                return _info.Result;
            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }
            return null;
        }

        public override async Task<cScanTimeInfo> GetScanTimeInfoAsync(cF4sdWebRequestInfo requestInfo, int LogDeep, CancellationToken Token)
        {
            return await GetScanTimeInfoAsync(Collector.InfrastructureConfig.ClientAgent.ScanTiming, "AgentScan-all", requestInfo, LogDeep, Token);
        }


        private async Task<List<structEventTableNames>> getDailyEventTables(int Days, cF4sdWebRequestInfo requestInfo, int LogDeep, CancellationToken Token)
        {
            return await getDailyEventTables(DateTime.UtcNow - TimeSpan.FromDays(Days), DateTime.MaxValue, requestInfo, LogDeep, Token);
        }


        private async Task<List<structEventTableNames>> getDailyEventTables(DateTime fromDate, DateTime toDate, cF4sdWebRequestInfo requestInfo, int LogDeep, CancellationToken Token)
        {
            var retVal = new List<structEventTableNames>();
            if (!useDailyEventTables)
                return retVal;

            await TableNamesCriticalSection.EnterAsync();
            try
            {
                var _now = DateTime.UtcNow;
                if (_now - EventTablesInfo.LastUpdate >= TimeSpan.FromMinutes(5))
                {
                    if (DataHistorySqlHelper.GetWellKnownSqlStatement("GetDailyAgentEventTables", out var Query))
                    {
                        var Params = new Dictionary<string, object>() { { "ORGCODE", OrganizationCode } };
                        // update the event table name list
                        MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
                        if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
                        var _startTime = DateTime.UtcNow;
                        int apiError = 0;

                        try
                        {
                            using (var Conn = new cDbConnection(Collector.InfrastructureConfig.ClientAgent.Connection))
                            {
                                using (var _reader = await DataHistorySqlHelper.GetDataReaderAsync(Conn, Query, Params, Token))
                                {
                                    if (_reader != null)
                                        try
                                        {
                                            if (_reader.HasRows)
                                            {
                                                EventTablesInfo.Clear();
                                                while (await _reader.ReadAsync())
                                                {
                                                    try
                                                    {
                                                        var _names = new structEventTableNames();
                                                        _names.From = _reader.GetDateTime(0);
                                                        _names.To = _reader.GetDateTime(1);
                                                        _names.SimpleValues = _reader.GetString(2);
                                                        _names.JsonData = _reader.GetString(3);
                                                        _names.DeviceActivity = _reader.GetString(4);
                                                        _names.UserActivity = _reader.GetString(5);
                                                        EventTablesInfo[_names.From] = _names;
                                                    }
                                                    catch (Exception E)
                                                    {
                                                        cLogManager.DefaultLogger.LogException(E);
                                                    }
                                                }
                                            }
                                        }
                                        catch (Exception E)
                                        {
                                            cLogManager.DefaultLogger.LogException(E);
                                        }
                                }
                            }
                        }
                        catch (Exception E)
                        {
                            apiError = E.HResult;
                            LogException(E);
                        }
                        finally
                        {
                            if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(Query.Name, _startTime, Params, apiError, requestInfo?.requestName);
                            if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                            if (CM != null) LogMethodEnd(CM);
                        }
                    }
                }

                foreach (var _entry in EventTablesInfo.Values)
                {
                    if (_entry.From <= toDate && _entry.To >= fromDate)
                        retVal.Insert(0, _entry);
                }
            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                TableNamesCriticalSection.Leave();
            }

            return retVal;
        }


        public async Task<List<cDataHistoryCollector.cUsageData>> GetSingleUsedInfoFromAgentData(enumFasdInformationClass infoClass, int AgentId, int Age, CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            try
            {
                var _listTasks = new List<Task<List<cDataHistoryCollector.cUsageData>>>();

                if (useOverallEventTables)
                {
                    if (!(EventTablesInfo?.Count >= 1) || EventTablesInfo.First().Value.From > DateTime.UtcNow - TimeSpan.FromDays(Age))
                        _listTasks.Add(GetSingleUsedInfoFromAgentDataTotal(infoClass, AgentId, Age, Token, requestInfo, LogDeep));
                }

                if (useDailyEventTables)
                {
                    var _tableNames = await getDailyEventTables(Age, requestInfo, LogDeep + 1, Token);
                    if (_tableNames != null)
                    {
                        foreach (var tableNameInfo in _tableNames)
                        {
                            _listTasks.Add(GetSingleUsedInfoFromAgentDataDaily(tableNameInfo.UserActivity, infoClass, AgentId, Token, requestInfo, LogDeep + 1));
                        }
                    }
                }

                var _taskResult = await Task.WhenAll(_listTasks).ConfigureAwait(false);
                if (_taskResult == null)
                    return null;

                var _dicResult = new Dictionary<int, cDataHistoryCollector.cUsageData>();

                foreach (var _resultItem in _taskResult)
                {
                    if (_resultItem == null)
                        continue;

                    foreach (var _itemEntry in _resultItem)
                    {
                        if (_dicResult.TryGetValue(_itemEntry.id, out var _usageInfo))
                        {
                            _usageInfo.duration += _itemEntry.duration;
                            if (_itemEntry.lastEvent > _usageInfo.lastEvent)
                                _usageInfo.lastEvent = _itemEntry.lastEvent;
                        }
                        else
                        {
                            _dicResult.Add(_itemEntry.id, _itemEntry);
                        }
                    }
                }

                return _dicResult.Values.ToList();
            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }

            return null;
        }


        private async Task<List<cDataHistoryCollector.cUsageData>> GetSingleUsedInfoFromAgentDataDaily(string TableName, enumFasdInformationClass infoClass, int AgentId, CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            if (!DataHistorySqlHelper.GetWellKnownSqlStatement($"Get{infoClass}UsageCountFromAgentDaily", out var Query))
                return null;

            var _strQuery = string.Format(Query.Query, TableName);
            Query = Query.Clone(_strQuery);

            var Params = new Dictionary<string, object>() { { "ID", AgentId } };
            int apiError = 0;

            try
            {
                var RetVal = new List<cDataHistoryCollector.cUsageData>();
                using (var ConnAgent = new cDbConnection(Collector.InfrastructureConfig.ClientAgent.Connection))
                using (var Reader = await DataHistorySqlHelper.GetDataReaderAsync(ConnAgent, Query, Params, Token))
                {
                    if (Reader != null)
                        try
                        {
                            if (Reader.HasRows)
                            {
                                while (await Reader.ReadAsync())
                                {
                                    try
                                    {
                                        var _entry = new cDataHistoryCollector.cUsageData
                                        {
                                            id = Reader.GetInt32(0),
                                            duration = Reader.GetInt64(1),
                                            lastEvent = Reader.GetDateTime(2)
                                        };
                                        RetVal.Add(_entry);
                                    }
                                    catch (Exception E)
                                    {
                                        cLogManager.DefaultLogger.LogException(E);
                                    }
                                }

                                if (RetVal.Count > 0)
                                    return RetVal;
                            }
                        }
                        catch (Exception E)
                        {
                            cLogManager.DefaultLogger.LogException(E);
                        }
                }
            }
            catch (Exception E)
            {
                apiError = E.HResult;
                LogException(E);
            }
            finally
            {
                if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(Query.Name, _startTime, Params, apiError, requestInfo?.requestName);
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }

            return null;

        }


        private async Task<List<cDataHistoryCollector.cUsageData>> GetSingleUsedInfoFromAgentDataTotal(enumFasdInformationClass infoClass, int AgentId, int Age, CancellationToken Token, cF4sdWebRequestInfo requestInfo, int LogDeep)
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }
            if (cPerformanceLogger.IsActive && requestInfo != null) { if (CM == null) CM = MethodBase.GetCurrentMethod(); cPerformanceLogger.LogPerformanceStart(LogDeep, CM, requestInfo.id, requestInfo.created); }
            var _startTime = DateTime.UtcNow;

            if (!DataHistorySqlHelper.GetWellKnownSqlStatement($"Get{infoClass}UsageCountFromAgent", out var Query))
                return null;

            var Params = new Dictionary<string, object>() { { "id", AgentId }, { "Age", Age } };
            int apiError = 0;

            try
            {
                var RetVal = new List<cDataHistoryCollector.cUsageData>();
                using (var ConnAgent = new cDbConnection(Collector.InfrastructureConfig.ClientAgent.Connection))
                using (var Reader = await DataHistorySqlHelper.GetDataReaderAsync(ConnAgent, Query, Params, Token))
                {
                    if (Reader != null)
                        try
                        {
                            if (Reader.HasRows)
                            {
                                while (await Reader.ReadAsync())
                                {
                                    try
                                    {
                                        var _entry = new cDataHistoryCollector.cUsageData
                                        {
                                            id = Reader.GetInt32(0),
                                            duration = Reader.GetInt64(1),
                                            lastEvent = Reader.GetDateTime(2)
                                        };
                                        RetVal.Add(_entry);
                                    }
                                    catch (Exception E)
                                    {
                                        cLogManager.DefaultLogger.LogException(E);
                                    }
                                }

                                if (RetVal.Count > 0)
                                    return RetVal;
                            }
                        }
                        catch (Exception E)
                        {
                            cLogManager.DefaultLogger.LogException(E);
                        }
                }
            }
            catch (Exception E)
            {
                apiError = E.HResult;
                LogException(E);
            }
            finally
            {
                if (DataHistorySqlHelper.LogSql) DataHistorySqlHelper.SaveSqlTimingEntry(Query.Name, _startTime, Params, apiError, requestInfo?.requestName);
                if (cPerformanceLogger.IsActive && requestInfo != null) { cPerformanceLogger.LogPerformanceEnd(LogDeep, CM, requestInfo.id, requestInfo.created, _startTime); }
                if (CM != null) LogMethodEnd(CM);
            }

            return null;
        }


        public cAgentApiConfiguration GetAgentApiConfiguration(bool decoded)
        {
            var AgentConfig = Collector?.InfrastructureConfig?.ClientAgent;
            if (AgentConfig == null)
                return null;

            var _retVal = new cAgentApiConfiguration()
            {
                ClientId = AgentConfig.ApiCredential.User,
                ApiUrl = new Uri(AgentConfig.ServerUri, AgentConfig.ApiPath).AbsoluteUri,
                LogonUrl = AgentConfig.ServerUri.AbsoluteUri,
                OrganizationCode = this.OrganizationCode
            };
            if (decoded)
                _retVal.ClientSecret = AgentConfig.ApiCredential.NwCredential.Password;
            else
                _retVal.ClientSecret = AgentConfig.ApiSecret;

            return _retVal;
        }

        public async Task<bool> LogonToAgentApiAsync()
        {
            var CM = MethodBase.GetCurrentMethod();
            LogMethodBegin(CM);
            try
            {
                return await _agentApiHelper.LogonAsync(GetAgentApiConfiguration(true));
            }
            catch (Exception ex)
            {
                LogException(ex);
            }
            finally
            {
                LogMethodEnd(CM);
            }

            return false;
        }

        public async Task<List<cAgentGetScriptResult>> GetRemoteScriptsAsync()
        {
            MethodBase CM = null; if (cLogManager.DefaultLogger.IsDebug) { CM = MethodBase.GetCurrentMethod(); LogMethodBegin(CM); }

            try
            {
                return await _agentApiHelper.GetScripts();
            }
            catch (Exception E)
            {
                LogException(E);
            }
            finally
            {
                if (CM != null) LogMethodEnd(CM);
            }

            return null;
        }

    }

    public class DuplicateKeyComparer<TKey>
                :
             IComparer<TKey> where TKey : IComparable
    {
        #region IComparer<TKey> Members

        public int Compare(TKey x, TKey y)
        {
            int result = x.CompareTo(y);

            if (result == 0)
                return 1;   // Handle equality as beeing greater
            else
                return result;
        }

        #endregion
    }

}
