From b095ca9749fbe9af2a935bcf3a965cac91f95826 Mon Sep 17 00:00:00 2001 From: Polianin Nikita Date: Sun, 27 Oct 2024 05:41:49 +0300 Subject: [PATCH] feat: add sync and mapper schedule --- Endpoint/Sync/Common/DataRepository.cs | 42 ++++ Endpoint/Sync/ScheduleSynchronizer.cs | 288 +++++++++++++++++++++++++ 2 files changed, 330 insertions(+) create mode 100644 Endpoint/Sync/Common/DataRepository.cs create mode 100644 Endpoint/Sync/ScheduleSynchronizer.cs diff --git a/Endpoint/Sync/Common/DataRepository.cs b/Endpoint/Sync/Common/DataRepository.cs new file mode 100644 index 0000000..c4de0b2 --- /dev/null +++ b/Endpoint/Sync/Common/DataRepository.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; + +namespace Mirea.Api.Endpoint.Sync.Common; + +internal class DataRepository where T : class +{ + private readonly ConcurrentBag _data = []; + private readonly object _lock = new(); + + public IEnumerable GetAll() => _data.ToList(); + + public DataRepository(List data) + { + foreach (var d in data) + _data.Add(d); + } + + public T? Get(Func predicate) + { + var entity = _data.FirstOrDefault(predicate); + return entity; + } + + public T Create(Func createEntity) + { + var entity = createEntity(); + _data.Add(entity); + return entity; + } + + public T GetOrCreate(Func predicate, Func createEntity) + { + lock (_lock) + { + var entity = Get(predicate); + return entity ?? Create(createEntity); + } + } +} \ No newline at end of file diff --git a/Endpoint/Sync/ScheduleSynchronizer.cs b/Endpoint/Sync/ScheduleSynchronizer.cs new file mode 100644 index 0000000..20d1e5d --- /dev/null +++ b/Endpoint/Sync/ScheduleSynchronizer.cs @@ -0,0 +1,288 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Mirea.Api.DataAccess.Domain.Schedule; +using Mirea.Api.DataAccess.Persistence; +using Mirea.Api.Endpoint.Common.Interfaces; +using Mirea.Api.Endpoint.Configuration.Model; +using Mirea.Api.Endpoint.Sync.Common; +using Mirea.Tools.Schedule.WebParser; +using Mirea.Tools.Schedule.WebParser.Common.Domain; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Group = Mirea.Api.DataAccess.Domain.Schedule.Group; + +namespace Mirea.Api.Endpoint.Sync; + +internal partial class ScheduleSynchronizer(UberDbContext dbContext, IOptionsSnapshot config, ILogger logger, IMaintenanceModeService maintenanceMode) +{ + private readonly DataRepository _campuses = new([.. dbContext.Campuses]); + private readonly DataRepository _disciplines = new([.. dbContext.Disciplines]); + private readonly DataRepository _faculties = new([.. dbContext.Faculties]); + private readonly DataRepository _groups = new([.. dbContext.Groups]); + private readonly DataRepository _lectureHalls = new([.. dbContext.LectureHalls]); + private readonly DataRepository _lessons = new([]); + private readonly DataRepository _lessonAssociation = new([]); + private readonly DataRepository _professors = new([.. dbContext.Professors]); + private readonly DataRepository _typeOfOccupations = new([.. dbContext.TypeOfOccupations]); + private readonly DataRepository _specificWeeks = new([]); + + // todo: transfer data to storage + private static string GetFaculty(char c) => + c switch + { + 'У' => "ИТУ", + 'Б' => "ИКБ", + 'Х' => "ИТХТ", + 'Э' => "ИПТИП", + 'Т' => "ИПТИП", + 'Р' => "ИРИ", + 'К' => "ИИИ", + 'И' => "ИИТ", + 'П' => "ИИТ", + _ => throw new ArgumentOutOfRangeException(nameof(c), c, null) + }; + + private void ParallelSync(GroupResult groupInfo) + { + var facultyName = GetFaculty(groupInfo.Group[0]); + + var faculty = _faculties.GetOrCreate( + f => f.Name.Equals(facultyName, StringComparison.OrdinalIgnoreCase), + () => new Faculty + { + Name = facultyName + }); + + var groupName = OnlyGroupName().Match(groupInfo.Group.ToUpper()).Value; + + var group = _groups.GetOrCreate( + g => g.Name.Equals(groupName, StringComparison.OrdinalIgnoreCase), + () => new Group + { + Name = groupName, + Faculty = faculty + }); + + var typeOfOccupation = _typeOfOccupations.GetOrCreate( + t => t.ShortName.Equals(groupInfo.TypeOfOccupation.Trim(), StringComparison.OrdinalIgnoreCase), + () => new TypeOfOccupation + { + ShortName = groupInfo.TypeOfOccupation.ToUpper() + }); + + List? professor = []; + if (groupInfo.Professor != null) + { + foreach (var prof in groupInfo.Professor) + { + var professorParts = prof.Split(' ').ToList(); + + string? altName = null; + + if (professorParts is { Count: >= 2 }) + { + altName = professorParts.ElementAtOrDefault(0); + + if (professorParts.ElementAtOrDefault(1) != null) + altName += $" {professorParts.ElementAtOrDefault(1)?[0]}."; + + if (professorParts.ElementAtOrDefault(2) != null) + altName += $"{professorParts.ElementAtOrDefault(2)?[0]}."; + } + + if (string.IsNullOrEmpty(altName)) + continue; + + var profDb = _professors.GetOrCreate(x => + (x.AltName == null || x.AltName.Equals(prof, StringComparison.OrdinalIgnoreCase)) && + x.Name.Equals(altName, StringComparison.OrdinalIgnoreCase), + () => new Professor + { + AltName = prof, + Name = altName + }); + + professor.Add(profDb); + } + } + else + professor = null; + + List? hall = null; + List? campuses; + if (groupInfo.Campuses != null && groupInfo.Campuses.Length != 0) + { + hall = []; + campuses = []; + for (int i = 0; i < groupInfo.Campuses.Length; i++) + { + var campus = groupInfo.Campuses[i]; + campuses.Add(_campuses.GetOrCreate( + c => c.CodeName.Equals(campus, StringComparison.OrdinalIgnoreCase), + () => new Campus + { + CodeName = campus.ToUpper() + })); + if (groupInfo.LectureHalls == null || groupInfo.LectureHalls.Length <= i) + continue; + var lectureHall = groupInfo.LectureHalls[i]; + + hall.Add(_lectureHalls.GetOrCreate(l => + l.Name.Equals(lectureHall, StringComparison.OrdinalIgnoreCase) && + string.Equals(l.Campus?.CodeName, campuses[^1].CodeName, StringComparison.CurrentCultureIgnoreCase), + () => new LectureHall + { + Name = lectureHall, + Campus = campuses[^1] + })); + } + } + + var discipline = _disciplines.GetOrCreate( + d => d.Name.Equals(groupInfo.Discipline, StringComparison.OrdinalIgnoreCase), + () => new Discipline + { + Name = groupInfo.Discipline + }); + + var lesson = _lessons.GetOrCreate(l => + l.IsEven == groupInfo.IsEven && + l.DayOfWeek == groupInfo.Day && + l.PairNumber == groupInfo.Pair && + l.Discipline?.Name == discipline.Name, + () => + { + var lesson = new Lesson + { + IsEven = groupInfo.IsEven, + DayOfWeek = groupInfo.Day, + PairNumber = groupInfo.Pair, + Discipline = discipline, + Group = group, + IsExcludedWeeks = groupInfo.IsExclude + }; + + if (groupInfo.SpecialWeek == null) + return lesson; + + foreach (var week in groupInfo.SpecialWeek) + _specificWeeks.Create(() => new SpecificWeek + { + Lesson = lesson, + WeekNumber = week + }); + + return lesson; + }); + + int maxValue = int.Max(int.Max(professor?.Count ?? -1, hall?.Count ?? -1), 1); + + for (int i = 0; i < maxValue; i++) + { + var prof = professor?.ElementAtOrDefault(i); + var lectureHall = hall?.ElementAtOrDefault(i); + _lessonAssociation.Create(() => new LessonAssociation + { + Professor = prof, + Lesson = lesson, + LectureHall = lectureHall, + TypeOfOccupation = typeOfOccupation + }); + } + } + + private async Task SaveChanges(CancellationToken cancellationToken) + { + foreach (var group in _groups.GetAll()) + { + var existingGroup = await dbContext.Groups.FirstOrDefaultAsync(g => g.Id == group.Id, cancellationToken); + if (existingGroup != null) + dbContext.Remove(existingGroup); + } + + await dbContext.Disciplines.BulkSynchronizeAsync(_disciplines.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken); + await dbContext.Professors.BulkSynchronizeAsync(_professors.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken); + await dbContext.TypeOfOccupations.BulkSynchronizeAsync(_typeOfOccupations.GetAll(), bulkOperation => bulkOperation.BatchSize = 100, cancellationToken); + await dbContext.Faculties.BulkSynchronizeAsync(_faculties.GetAll(), bulkOperation => bulkOperation.BatchSize = 100, cancellationToken); + await dbContext.Campuses.BulkSynchronizeAsync(_campuses.GetAll(), bulkOperation => bulkOperation.BatchSize = 10, cancellationToken); + await dbContext.LectureHalls.BulkSynchronizeAsync(_lectureHalls.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken); + await dbContext.Groups.BulkSynchronizeAsync(_groups.GetAll(), bulkOperation => bulkOperation.BatchSize = 100, cancellationToken); + await dbContext.Lessons.BulkSynchronizeAsync(_lessons.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken); + await dbContext.SpecificWeeks.BulkSynchronizeAsync(_specificWeeks.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken); + await dbContext.LessonAssociations.BulkSynchronizeAsync(_lessonAssociation.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken); + } + + public async Task StartSync(CancellationToken cancellationToken) + { + var pairPeriods = config.Value.ScheduleSettings?.PairPeriod; + var startTerm = config.Value.ScheduleSettings?.StartTerm; + + if (pairPeriods == null || startTerm == null) + { + logger.LogWarning("It is not possible to synchronize the schedule due to the fact that the {Arg1} or {Arg2} variable is not initialized.", nameof(pairPeriods), nameof(startTerm)); + + return; + } + + Stopwatch watch = new(); + watch.Start(); + + var parser = new Parser + { + Pairs = pairPeriods + .ToDictionary(x => x.Key, + x => (x.Value.Start, x.Value.End)), + TermStart = startTerm.Value.ToDateTime(new TimeOnly(0, 0, 0)) + }; + + try + { + logger.LogDebug("Start parsing schedule"); + var data = await parser.ParseAsync(cancellationToken); + + watch.Stop(); + var parsingTime = watch.ElapsedMilliseconds; + watch.Restart(); + + ParallelOptions options = new() + { + CancellationToken = cancellationToken, + MaxDegreeOfParallelism = Environment.ProcessorCount + }; + + logger.LogDebug("Start mapping parsed data"); + Parallel.ForEach(data, options, ParallelSync); + + watch.Stop(); + var mappingTime = watch.ElapsedMilliseconds; + watch.Restart(); + + maintenanceMode.EnableMaintenanceMode(); + + logger.LogDebug("Start saving changing"); + await SaveChanges(cancellationToken); + + maintenanceMode.DisableMaintenanceMode(); + + watch.Stop(); + + logger.LogInformation("Parsing time: {ParsingTime}ms Mapping time: {MappingTime}ms Saving time: {SavingTime}ms Total time: {TotalTime}ms", + parsingTime, mappingTime, watch.ElapsedMilliseconds, parsingTime + mappingTime + watch.ElapsedMilliseconds); + } + catch (Exception ex) + { + logger.LogError(ex, "An error occurred during synchronization."); + maintenanceMode.DisableMaintenanceMode(); + throw; + } + } + + [GeneratedRegex(@"\w{4}-\d{2}-\d{2}(?=\s?\d?\s?[Пп]/?[Гг]\s?\d?)?")] + private static partial Regex OnlyGroupName(); +} \ No newline at end of file