feat: add sync and mapper schedule

Endpoint/Sync/Common/DataRepository.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Mirea.Api.Endpoint.Sync.Common;
+
+internal class DataRepository<T> where T : class
+{
+    private readonly ConcurrentBag<T> _data = [];
+    private readonly object _lock = new();
+
+    public IEnumerable<T> GetAll() => _data.ToList();
+
+    public DataRepository(List<T> data)
+    {
+        foreach (var d in data)
+            _data.Add(d);
+    }
+
+    public T? Get(Func<T, bool> predicate)
+    {
+        var entity = _data.FirstOrDefault(predicate);
+        return entity;
+    }
+
+    public T Create(Func<T> createEntity)
+    {
+        var entity = createEntity();
+        _data.Add(entity);
+        return entity;
+    }
+
+    public T GetOrCreate(Func<T, bool> predicate, Func<T> createEntity)
+    {
+        lock (_lock)
+        {
+            var entity = Get(predicate);
+            return entity ?? Create(createEntity);
+        }
+    }
+}

Endpoint/Sync/ScheduleSynchronizer.cs

@@ -0,0 +1,288 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Mirea.Api.DataAccess.Domain.Schedule;
+using Mirea.Api.DataAccess.Persistence;
+using Mirea.Api.Endpoint.Common.Interfaces;
+using Mirea.Api.Endpoint.Configuration.Model;
+using Mirea.Api.Endpoint.Sync.Common;
+using Mirea.Tools.Schedule.WebParser;
+using Mirea.Tools.Schedule.WebParser.Common.Domain;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using Group = Mirea.Api.DataAccess.Domain.Schedule.Group;
+
+namespace Mirea.Api.Endpoint.Sync;
+
+internal partial class ScheduleSynchronizer(UberDbContext dbContext, IOptionsSnapshot<GeneralConfig> config, ILogger<ScheduleSynchronizer> logger, IMaintenanceModeService maintenanceMode)
+{
+    private readonly DataRepository<Campus> _campuses = new([.. dbContext.Campuses]);
+    private readonly DataRepository<Discipline> _disciplines = new([.. dbContext.Disciplines]);
+    private readonly DataRepository<Faculty> _faculties = new([.. dbContext.Faculties]);
+    private readonly DataRepository<Group> _groups = new([.. dbContext.Groups]);
+    private readonly DataRepository<LectureHall> _lectureHalls = new([.. dbContext.LectureHalls]);
+    private readonly DataRepository<Lesson> _lessons = new([]);
+    private readonly DataRepository<LessonAssociation> _lessonAssociation = new([]);
+    private readonly DataRepository<Professor> _professors = new([.. dbContext.Professors]);
+    private readonly DataRepository<TypeOfOccupation> _typeOfOccupations = new([.. dbContext.TypeOfOccupations]);
+    private readonly DataRepository<SpecificWeek> _specificWeeks = new([]);
+
+    // todo: transfer data to storage
+    private static string GetFaculty(char c) =>
+        c switch
+        {
+            'У' => "ИТУ",
+            'Б' => "ИКБ",
+            'Х' => "ИТХТ",
+            'Э' => "ИПТИП",
+            'Т' => "ИПТИП",
+            'Р' => "ИРИ",
+            'К' => "ИИИ",
+            'И' => "ИИТ",
+            'П' => "ИИТ",
+            _ => throw new ArgumentOutOfRangeException(nameof(c), c, null)
+        };
+
+    private void ParallelSync(GroupResult groupInfo)
+    {
+        var facultyName = GetFaculty(groupInfo.Group[0]);
+
+        var faculty = _faculties.GetOrCreate(
+            f => f.Name.Equals(facultyName, StringComparison.OrdinalIgnoreCase),
+            () => new Faculty
+            {
+                Name = facultyName
+            });
+
+        var groupName = OnlyGroupName().Match(groupInfo.Group.ToUpper()).Value;
+
+        var group = _groups.GetOrCreate(
+            g => g.Name.Equals(groupName, StringComparison.OrdinalIgnoreCase),
+            () => new Group
+            {
+                Name = groupName,
+                Faculty = faculty
+            });
+
+        var typeOfOccupation = _typeOfOccupations.GetOrCreate(
+            t => t.ShortName.Equals(groupInfo.TypeOfOccupation.Trim(), StringComparison.OrdinalIgnoreCase),
+            () => new TypeOfOccupation
+            {
+                ShortName = groupInfo.TypeOfOccupation.ToUpper()
+            });
+
+        List<Professor>? professor = [];
+        if (groupInfo.Professor != null)
+        {
+            foreach (var prof in groupInfo.Professor)
+            {
+                var professorParts = prof.Split(' ').ToList();
+
+                string? altName = null;
+
+                if (professorParts is { Count: >= 2 })
+                {
+                    altName = professorParts.ElementAtOrDefault(0);
+
+                    if (professorParts.ElementAtOrDefault(1) != null)
+                        altName += $" {professorParts.ElementAtOrDefault(1)?[0]}.";
+
+                    if (professorParts.ElementAtOrDefault(2) != null)
+                        altName += $"{professorParts.ElementAtOrDefault(2)?[0]}.";
+                }
+
+                if (string.IsNullOrEmpty(altName))
+                    continue;
+
+                var profDb = _professors.GetOrCreate(x =>
+                        (x.AltName == null || x.AltName.Equals(prof, StringComparison.OrdinalIgnoreCase)) &&
+                        x.Name.Equals(altName, StringComparison.OrdinalIgnoreCase),
+                    () => new Professor
+                    {
+                        AltName = prof,
+                        Name = altName
+                    });
+
+                professor.Add(profDb);
+            }
+        }
+        else
+            professor = null;
+
+        List<LectureHall>? hall = null;
+        List<Campus>? campuses;
+        if (groupInfo.Campuses != null && groupInfo.Campuses.Length != 0)
+        {
+            hall = [];
+            campuses = [];
+            for (int i = 0; i < groupInfo.Campuses.Length; i++)
+            {
+                var campus = groupInfo.Campuses[i];
+                campuses.Add(_campuses.GetOrCreate(
+                    c => c.CodeName.Equals(campus, StringComparison.OrdinalIgnoreCase),
+                    () => new Campus
+                    {
+                        CodeName = campus.ToUpper()
+                    }));
+                if (groupInfo.LectureHalls == null || groupInfo.LectureHalls.Length <= i)
+                    continue;
+                var lectureHall = groupInfo.LectureHalls[i];
+
+                hall.Add(_lectureHalls.GetOrCreate(l =>
+                        l.Name.Equals(lectureHall, StringComparison.OrdinalIgnoreCase) &&
+                        string.Equals(l.Campus?.CodeName, campuses[^1].CodeName, StringComparison.CurrentCultureIgnoreCase),
+                    () => new LectureHall
+                    {
+                        Name = lectureHall,
+                        Campus = campuses[^1]
+                    }));
+            }
+        }
+
+        var discipline = _disciplines.GetOrCreate(
+            d => d.Name.Equals(groupInfo.Discipline, StringComparison.OrdinalIgnoreCase),
+            () => new Discipline
+            {
+                Name = groupInfo.Discipline
+            });
+
+        var lesson = _lessons.GetOrCreate(l =>
+                l.IsEven == groupInfo.IsEven &&
+                l.DayOfWeek == groupInfo.Day &&
+                l.PairNumber == groupInfo.Pair &&
+                l.Discipline?.Name == discipline.Name,
+            () =>
+            {
+                var lesson = new Lesson
+                {
+                    IsEven = groupInfo.IsEven,
+                    DayOfWeek = groupInfo.Day,
+                    PairNumber = groupInfo.Pair,
+                    Discipline = discipline,
+                    Group = group,
+                    IsExcludedWeeks = groupInfo.IsExclude
+                };
+
+                if (groupInfo.SpecialWeek == null)
+                    return lesson;
+
+                foreach (var week in groupInfo.SpecialWeek)
+                    _specificWeeks.Create(() => new SpecificWeek
+                    {
+                        Lesson = lesson,
+                        WeekNumber = week
+                    });
+
+                return lesson;
+            });
+
+        int maxValue = int.Max(int.Max(professor?.Count ?? -1, hall?.Count ?? -1), 1);
+
+        for (int i = 0; i < maxValue; i++)
+        {
+            var prof = professor?.ElementAtOrDefault(i);
+            var lectureHall = hall?.ElementAtOrDefault(i);
+            _lessonAssociation.Create(() => new LessonAssociation
+            {
+                Professor = prof,
+                Lesson = lesson,
+                LectureHall = lectureHall,
+                TypeOfOccupation = typeOfOccupation
+            });
+        }
+    }
+
+    private async Task SaveChanges(CancellationToken cancellationToken)
+    {
+        foreach (var group in _groups.GetAll())
+        {
+            var existingGroup = await dbContext.Groups.FirstOrDefaultAsync(g => g.Id == group.Id, cancellationToken);
+            if (existingGroup != null)
+                dbContext.Remove(existingGroup);
+        }
+
+        await dbContext.Disciplines.BulkSynchronizeAsync(_disciplines.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
+        await dbContext.Professors.BulkSynchronizeAsync(_professors.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
+        await dbContext.TypeOfOccupations.BulkSynchronizeAsync(_typeOfOccupations.GetAll(), bulkOperation => bulkOperation.BatchSize = 100, cancellationToken);
+        await dbContext.Faculties.BulkSynchronizeAsync(_faculties.GetAll(), bulkOperation => bulkOperation.BatchSize = 100, cancellationToken);
+        await dbContext.Campuses.BulkSynchronizeAsync(_campuses.GetAll(), bulkOperation => bulkOperation.BatchSize = 10, cancellationToken);
+        await dbContext.LectureHalls.BulkSynchronizeAsync(_lectureHalls.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
+        await dbContext.Groups.BulkSynchronizeAsync(_groups.GetAll(), bulkOperation => bulkOperation.BatchSize = 100, cancellationToken);
+        await dbContext.Lessons.BulkSynchronizeAsync(_lessons.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
+        await dbContext.SpecificWeeks.BulkSynchronizeAsync(_specificWeeks.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
+        await dbContext.LessonAssociations.BulkSynchronizeAsync(_lessonAssociation.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
+    }
+
+    public async Task StartSync(CancellationToken cancellationToken)
+    {
+        var pairPeriods = config.Value.ScheduleSettings?.PairPeriod;
+        var startTerm = config.Value.ScheduleSettings?.StartTerm;
+
+        if (pairPeriods == null || startTerm == null)
+        {
+            logger.LogWarning("It is not possible to synchronize the schedule due to the fact that the {Arg1} or {Arg2} variable is not initialized.", nameof(pairPeriods), nameof(startTerm));
+
+            return;
+        }
+
+        Stopwatch watch = new();
+        watch.Start();
+
+        var parser = new Parser
+        {
+            Pairs = pairPeriods
+                .ToDictionary(x => x.Key,
+                    x => (x.Value.Start, x.Value.End)),
+            TermStart = startTerm.Value.ToDateTime(new TimeOnly(0, 0, 0))
+        };
+
+        try
+        {
+            logger.LogDebug("Start parsing schedule");
+            var data = await parser.ParseAsync(cancellationToken);
+
+            watch.Stop();
+            var parsingTime = watch.ElapsedMilliseconds;
+            watch.Restart();
+
+            ParallelOptions options = new()
+            {
+                CancellationToken = cancellationToken,
+                MaxDegreeOfParallelism = Environment.ProcessorCount
+            };
+
+            logger.LogDebug("Start mapping parsed data");
+            Parallel.ForEach(data, options, ParallelSync);
+
+            watch.Stop();
+            var mappingTime = watch.ElapsedMilliseconds;
+            watch.Restart();
+
+            maintenanceMode.EnableMaintenanceMode();
+
+            logger.LogDebug("Start saving changing");
+            await SaveChanges(cancellationToken);
+
+            maintenanceMode.DisableMaintenanceMode();
+
+            watch.Stop();
+
+            logger.LogInformation("Parsing time: {ParsingTime}ms Mapping time: {MappingTime}ms Saving time: {SavingTime}ms Total time: {TotalTime}ms",
+                parsingTime, mappingTime, watch.ElapsedMilliseconds, parsingTime + mappingTime + watch.ElapsedMilliseconds);
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "An error occurred during synchronization.");
+            maintenanceMode.DisableMaintenanceMode();
+            throw;
+        }
+    }
+
+    [GeneratedRegex(@"\w{4}-\d{2}-\d{2}(?=\s?\d?\s?[Пп]/?[Гг]\s?\d?)?")]
+    private static partial Regex OnlyGroupName();
+}