feat: add sync and mapper schedule
This commit is contained in:
parent
8fad070a9c
commit
b095ca9749
42
Endpoint/Sync/Common/DataRepository.cs
Normal file
42
Endpoint/Sync/Common/DataRepository.cs
Normal file
@ -0,0 +1,42 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace Mirea.Api.Endpoint.Sync.Common;
|
||||
|
||||
internal class DataRepository<T> where T : class
|
||||
{
|
||||
private readonly ConcurrentBag<T> _data = [];
|
||||
private readonly object _lock = new();
|
||||
|
||||
public IEnumerable<T> GetAll() => _data.ToList();
|
||||
|
||||
public DataRepository(List<T> data)
|
||||
{
|
||||
foreach (var d in data)
|
||||
_data.Add(d);
|
||||
}
|
||||
|
||||
public T? Get(Func<T, bool> predicate)
|
||||
{
|
||||
var entity = _data.FirstOrDefault(predicate);
|
||||
return entity;
|
||||
}
|
||||
|
||||
public T Create(Func<T> createEntity)
|
||||
{
|
||||
var entity = createEntity();
|
||||
_data.Add(entity);
|
||||
return entity;
|
||||
}
|
||||
|
||||
public T GetOrCreate(Func<T, bool> predicate, Func<T> createEntity)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
var entity = Get(predicate);
|
||||
return entity ?? Create(createEntity);
|
||||
}
|
||||
}
|
||||
}
|
||||
288
Endpoint/Sync/ScheduleSynchronizer.cs
Normal file
288
Endpoint/Sync/ScheduleSynchronizer.cs
Normal file
@ -0,0 +1,288 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Mirea.Api.DataAccess.Domain.Schedule;
|
||||
using Mirea.Api.DataAccess.Persistence;
|
||||
using Mirea.Api.Endpoint.Common.Interfaces;
|
||||
using Mirea.Api.Endpoint.Configuration.Model;
|
||||
using Mirea.Api.Endpoint.Sync.Common;
|
||||
using Mirea.Tools.Schedule.WebParser;
|
||||
using Mirea.Tools.Schedule.WebParser.Common.Domain;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Group = Mirea.Api.DataAccess.Domain.Schedule.Group;
|
||||
|
||||
namespace Mirea.Api.Endpoint.Sync;
|
||||
|
||||
internal partial class ScheduleSynchronizer(UberDbContext dbContext, IOptionsSnapshot<GeneralConfig> config, ILogger<ScheduleSynchronizer> logger, IMaintenanceModeService maintenanceMode)
|
||||
{
|
||||
private readonly DataRepository<Campus> _campuses = new([.. dbContext.Campuses]);
|
||||
private readonly DataRepository<Discipline> _disciplines = new([.. dbContext.Disciplines]);
|
||||
private readonly DataRepository<Faculty> _faculties = new([.. dbContext.Faculties]);
|
||||
private readonly DataRepository<Group> _groups = new([.. dbContext.Groups]);
|
||||
private readonly DataRepository<LectureHall> _lectureHalls = new([.. dbContext.LectureHalls]);
|
||||
private readonly DataRepository<Lesson> _lessons = new([]);
|
||||
private readonly DataRepository<LessonAssociation> _lessonAssociation = new([]);
|
||||
private readonly DataRepository<Professor> _professors = new([.. dbContext.Professors]);
|
||||
private readonly DataRepository<TypeOfOccupation> _typeOfOccupations = new([.. dbContext.TypeOfOccupations]);
|
||||
private readonly DataRepository<SpecificWeek> _specificWeeks = new([]);
|
||||
|
||||
// todo: transfer data to storage
|
||||
private static string GetFaculty(char c) =>
|
||||
c switch
|
||||
{
|
||||
'У' => "ИТУ",
|
||||
'Б' => "ИКБ",
|
||||
'Х' => "ИТХТ",
|
||||
'Э' => "ИПТИП",
|
||||
'Т' => "ИПТИП",
|
||||
'Р' => "ИРИ",
|
||||
'К' => "ИИИ",
|
||||
'И' => "ИИТ",
|
||||
'П' => "ИИТ",
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(c), c, null)
|
||||
};
|
||||
|
||||
private void ParallelSync(GroupResult groupInfo)
|
||||
{
|
||||
var facultyName = GetFaculty(groupInfo.Group[0]);
|
||||
|
||||
var faculty = _faculties.GetOrCreate(
|
||||
f => f.Name.Equals(facultyName, StringComparison.OrdinalIgnoreCase),
|
||||
() => new Faculty
|
||||
{
|
||||
Name = facultyName
|
||||
});
|
||||
|
||||
var groupName = OnlyGroupName().Match(groupInfo.Group.ToUpper()).Value;
|
||||
|
||||
var group = _groups.GetOrCreate(
|
||||
g => g.Name.Equals(groupName, StringComparison.OrdinalIgnoreCase),
|
||||
() => new Group
|
||||
{
|
||||
Name = groupName,
|
||||
Faculty = faculty
|
||||
});
|
||||
|
||||
var typeOfOccupation = _typeOfOccupations.GetOrCreate(
|
||||
t => t.ShortName.Equals(groupInfo.TypeOfOccupation.Trim(), StringComparison.OrdinalIgnoreCase),
|
||||
() => new TypeOfOccupation
|
||||
{
|
||||
ShortName = groupInfo.TypeOfOccupation.ToUpper()
|
||||
});
|
||||
|
||||
List<Professor>? professor = [];
|
||||
if (groupInfo.Professor != null)
|
||||
{
|
||||
foreach (var prof in groupInfo.Professor)
|
||||
{
|
||||
var professorParts = prof.Split(' ').ToList();
|
||||
|
||||
string? altName = null;
|
||||
|
||||
if (professorParts is { Count: >= 2 })
|
||||
{
|
||||
altName = professorParts.ElementAtOrDefault(0);
|
||||
|
||||
if (professorParts.ElementAtOrDefault(1) != null)
|
||||
altName += $" {professorParts.ElementAtOrDefault(1)?[0]}.";
|
||||
|
||||
if (professorParts.ElementAtOrDefault(2) != null)
|
||||
altName += $"{professorParts.ElementAtOrDefault(2)?[0]}.";
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(altName))
|
||||
continue;
|
||||
|
||||
var profDb = _professors.GetOrCreate(x =>
|
||||
(x.AltName == null || x.AltName.Equals(prof, StringComparison.OrdinalIgnoreCase)) &&
|
||||
x.Name.Equals(altName, StringComparison.OrdinalIgnoreCase),
|
||||
() => new Professor
|
||||
{
|
||||
AltName = prof,
|
||||
Name = altName
|
||||
});
|
||||
|
||||
professor.Add(profDb);
|
||||
}
|
||||
}
|
||||
else
|
||||
professor = null;
|
||||
|
||||
List<LectureHall>? hall = null;
|
||||
List<Campus>? campuses;
|
||||
if (groupInfo.Campuses != null && groupInfo.Campuses.Length != 0)
|
||||
{
|
||||
hall = [];
|
||||
campuses = [];
|
||||
for (int i = 0; i < groupInfo.Campuses.Length; i++)
|
||||
{
|
||||
var campus = groupInfo.Campuses[i];
|
||||
campuses.Add(_campuses.GetOrCreate(
|
||||
c => c.CodeName.Equals(campus, StringComparison.OrdinalIgnoreCase),
|
||||
() => new Campus
|
||||
{
|
||||
CodeName = campus.ToUpper()
|
||||
}));
|
||||
if (groupInfo.LectureHalls == null || groupInfo.LectureHalls.Length <= i)
|
||||
continue;
|
||||
var lectureHall = groupInfo.LectureHalls[i];
|
||||
|
||||
hall.Add(_lectureHalls.GetOrCreate(l =>
|
||||
l.Name.Equals(lectureHall, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(l.Campus?.CodeName, campuses[^1].CodeName, StringComparison.CurrentCultureIgnoreCase),
|
||||
() => new LectureHall
|
||||
{
|
||||
Name = lectureHall,
|
||||
Campus = campuses[^1]
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
var discipline = _disciplines.GetOrCreate(
|
||||
d => d.Name.Equals(groupInfo.Discipline, StringComparison.OrdinalIgnoreCase),
|
||||
() => new Discipline
|
||||
{
|
||||
Name = groupInfo.Discipline
|
||||
});
|
||||
|
||||
var lesson = _lessons.GetOrCreate(l =>
|
||||
l.IsEven == groupInfo.IsEven &&
|
||||
l.DayOfWeek == groupInfo.Day &&
|
||||
l.PairNumber == groupInfo.Pair &&
|
||||
l.Discipline?.Name == discipline.Name,
|
||||
() =>
|
||||
{
|
||||
var lesson = new Lesson
|
||||
{
|
||||
IsEven = groupInfo.IsEven,
|
||||
DayOfWeek = groupInfo.Day,
|
||||
PairNumber = groupInfo.Pair,
|
||||
Discipline = discipline,
|
||||
Group = group,
|
||||
IsExcludedWeeks = groupInfo.IsExclude
|
||||
};
|
||||
|
||||
if (groupInfo.SpecialWeek == null)
|
||||
return lesson;
|
||||
|
||||
foreach (var week in groupInfo.SpecialWeek)
|
||||
_specificWeeks.Create(() => new SpecificWeek
|
||||
{
|
||||
Lesson = lesson,
|
||||
WeekNumber = week
|
||||
});
|
||||
|
||||
return lesson;
|
||||
});
|
||||
|
||||
int maxValue = int.Max(int.Max(professor?.Count ?? -1, hall?.Count ?? -1), 1);
|
||||
|
||||
for (int i = 0; i < maxValue; i++)
|
||||
{
|
||||
var prof = professor?.ElementAtOrDefault(i);
|
||||
var lectureHall = hall?.ElementAtOrDefault(i);
|
||||
_lessonAssociation.Create(() => new LessonAssociation
|
||||
{
|
||||
Professor = prof,
|
||||
Lesson = lesson,
|
||||
LectureHall = lectureHall,
|
||||
TypeOfOccupation = typeOfOccupation
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SaveChanges(CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var group in _groups.GetAll())
|
||||
{
|
||||
var existingGroup = await dbContext.Groups.FirstOrDefaultAsync(g => g.Id == group.Id, cancellationToken);
|
||||
if (existingGroup != null)
|
||||
dbContext.Remove(existingGroup);
|
||||
}
|
||||
|
||||
await dbContext.Disciplines.BulkSynchronizeAsync(_disciplines.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
|
||||
await dbContext.Professors.BulkSynchronizeAsync(_professors.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
|
||||
await dbContext.TypeOfOccupations.BulkSynchronizeAsync(_typeOfOccupations.GetAll(), bulkOperation => bulkOperation.BatchSize = 100, cancellationToken);
|
||||
await dbContext.Faculties.BulkSynchronizeAsync(_faculties.GetAll(), bulkOperation => bulkOperation.BatchSize = 100, cancellationToken);
|
||||
await dbContext.Campuses.BulkSynchronizeAsync(_campuses.GetAll(), bulkOperation => bulkOperation.BatchSize = 10, cancellationToken);
|
||||
await dbContext.LectureHalls.BulkSynchronizeAsync(_lectureHalls.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
|
||||
await dbContext.Groups.BulkSynchronizeAsync(_groups.GetAll(), bulkOperation => bulkOperation.BatchSize = 100, cancellationToken);
|
||||
await dbContext.Lessons.BulkSynchronizeAsync(_lessons.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
|
||||
await dbContext.SpecificWeeks.BulkSynchronizeAsync(_specificWeeks.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
|
||||
await dbContext.LessonAssociations.BulkSynchronizeAsync(_lessonAssociation.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task StartSync(CancellationToken cancellationToken)
|
||||
{
|
||||
var pairPeriods = config.Value.ScheduleSettings?.PairPeriod;
|
||||
var startTerm = config.Value.ScheduleSettings?.StartTerm;
|
||||
|
||||
if (pairPeriods == null || startTerm == null)
|
||||
{
|
||||
logger.LogWarning("It is not possible to synchronize the schedule due to the fact that the {Arg1} or {Arg2} variable is not initialized.", nameof(pairPeriods), nameof(startTerm));
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
Stopwatch watch = new();
|
||||
watch.Start();
|
||||
|
||||
var parser = new Parser
|
||||
{
|
||||
Pairs = pairPeriods
|
||||
.ToDictionary(x => x.Key,
|
||||
x => (x.Value.Start, x.Value.End)),
|
||||
TermStart = startTerm.Value.ToDateTime(new TimeOnly(0, 0, 0))
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
logger.LogDebug("Start parsing schedule");
|
||||
var data = await parser.ParseAsync(cancellationToken);
|
||||
|
||||
watch.Stop();
|
||||
var parsingTime = watch.ElapsedMilliseconds;
|
||||
watch.Restart();
|
||||
|
||||
ParallelOptions options = new()
|
||||
{
|
||||
CancellationToken = cancellationToken,
|
||||
MaxDegreeOfParallelism = Environment.ProcessorCount
|
||||
};
|
||||
|
||||
logger.LogDebug("Start mapping parsed data");
|
||||
Parallel.ForEach(data, options, ParallelSync);
|
||||
|
||||
watch.Stop();
|
||||
var mappingTime = watch.ElapsedMilliseconds;
|
||||
watch.Restart();
|
||||
|
||||
maintenanceMode.EnableMaintenanceMode();
|
||||
|
||||
logger.LogDebug("Start saving changing");
|
||||
await SaveChanges(cancellationToken);
|
||||
|
||||
maintenanceMode.DisableMaintenanceMode();
|
||||
|
||||
watch.Stop();
|
||||
|
||||
logger.LogInformation("Parsing time: {ParsingTime}ms Mapping time: {MappingTime}ms Saving time: {SavingTime}ms Total time: {TotalTime}ms",
|
||||
parsingTime, mappingTime, watch.ElapsedMilliseconds, parsingTime + mappingTime + watch.ElapsedMilliseconds);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "An error occurred during synchronization.");
|
||||
maintenanceMode.DisableMaintenanceMode();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
[GeneratedRegex(@"\w{4}-\d{2}-\d{2}(?=\s?\d?\s?[Пп]/?[Гг]\s?\d?)?")]
|
||||
private static partial Regex OnlyGroupName();
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user