518 lines
19 KiB
C#
518 lines
19 KiB
C#
using Microsoft.EntityFrameworkCore;
|
||
using Microsoft.Extensions.Logging;
|
||
using Microsoft.Extensions.Options;
|
||
using Mirea.Api.DataAccess.Domain.Schedule;
|
||
using Mirea.Api.DataAccess.Persistence;
|
||
using Mirea.Api.Endpoint.Common.Interfaces;
|
||
using Mirea.Api.Endpoint.Configuration.Model;
|
||
using Mirea.Api.Endpoint.Sync.Common;
|
||
using Mirea.Tools.Schedule.Parser.Domain;
|
||
using Mirea.Tools.Schedule.WebParser;
|
||
using Mirea.Tools.Schedule.WebParser.Common.Domain;
|
||
using System;
|
||
using System.Collections.Concurrent;
|
||
using System.Collections.Generic;
|
||
using System.Diagnostics;
|
||
using System.Linq;
|
||
using System.Text.RegularExpressions;
|
||
using System.Threading;
|
||
using System.Threading.Tasks;
|
||
using Group = Mirea.Api.DataAccess.Domain.Schedule.Group;
|
||
|
||
namespace Mirea.Api.Endpoint.Sync;
|
||
|
||
internal partial class ScheduleSynchronizer(UberDbContext dbContext, IOptionsSnapshot<GeneralConfig> config, ILogger<ScheduleSynchronizer> logger,
|
||
IMaintenanceModeService maintenanceMode)
|
||
{
|
||
private readonly DataRepository<Campus> _campuses = new([.. dbContext.Campuses]);
|
||
private readonly DataRepository<Discipline> _disciplines = new([.. dbContext.Disciplines]);
|
||
private readonly DataRepository<Faculty> _faculties = new([.. dbContext.Faculties]);
|
||
private readonly DataRepository<Group> _groups = new([.. dbContext.Groups]);
|
||
private readonly DataRepository<LectureHall> _lectureHalls = new([.. dbContext.LectureHalls]);
|
||
private readonly DataRepository<Lesson> _lessons = new([]);
|
||
private readonly DataRepository<LessonAssociation> _lessonAssociation = new([]);
|
||
private readonly DataRepository<Professor> _professors = new([.. dbContext.Professors]);
|
||
private readonly DataRepository<TypeOfOccupation> _typeOfOccupations = new([.. dbContext.TypeOfOccupations]);
|
||
private readonly DataRepository<SpecificWeek> _specificWeeks = new([]);
|
||
|
||
// todo: transfer data to storage
|
||
private static string GetFaculty(char c) =>
|
||
c switch
|
||
{
|
||
'У' => "ИТУ",
|
||
'Б' => "ИКБ",
|
||
'Х' => "ИТХТ",
|
||
'Э' => "ИПТИП",
|
||
'Т' => "ИПТИП",
|
||
'Р' => "ИРИ",
|
||
'К' => "ИИИ",
|
||
'И' => "ИИТ",
|
||
'П' => "ИИТ",
|
||
_ => throw new ArgumentOutOfRangeException(nameof(c), c, null)
|
||
};
|
||
|
||
private void ParallelSync(GroupResult groupInfo)
|
||
{
|
||
var facultyName = GetFaculty(groupInfo.Group[0]);
|
||
|
||
var faculty = _faculties.GetOrCreate(
|
||
f => f.Name.Equals(facultyName, StringComparison.OrdinalIgnoreCase),
|
||
() => new Faculty
|
||
{
|
||
Name = facultyName
|
||
});
|
||
|
||
var groupName = OnlyGroupName().Match(groupInfo.Group.ToUpper()).Value;
|
||
|
||
var group = _groups.GetOrCreate(
|
||
g => g.Name.Equals(groupName, StringComparison.OrdinalIgnoreCase),
|
||
() => new Group
|
||
{
|
||
Name = groupName,
|
||
Faculty = faculty
|
||
});
|
||
|
||
var typeOfOccupation = _typeOfOccupations.GetOrCreate(
|
||
t => t.ShortName.Equals(groupInfo.TypeOfOccupation.Trim(), StringComparison.OrdinalIgnoreCase),
|
||
() => new TypeOfOccupation
|
||
{
|
||
ShortName = groupInfo.TypeOfOccupation.ToUpper()
|
||
});
|
||
|
||
List<Professor>? professor = [];
|
||
if (groupInfo.Professor != null)
|
||
{
|
||
foreach (var prof in groupInfo.Professor)
|
||
{
|
||
var professorParts = prof.Split(' ').ToList();
|
||
|
||
string? altName = null;
|
||
|
||
if (professorParts is { Count: >= 2 })
|
||
{
|
||
altName = professorParts.ElementAtOrDefault(0);
|
||
|
||
if (professorParts.ElementAtOrDefault(1) != null)
|
||
altName += $" {professorParts.ElementAtOrDefault(1)?[0]}.";
|
||
|
||
if (professorParts.ElementAtOrDefault(2) != null)
|
||
altName += $"{professorParts.ElementAtOrDefault(2)?[0]}.";
|
||
}
|
||
|
||
if (string.IsNullOrEmpty(altName))
|
||
continue;
|
||
|
||
var profDb = _professors.GetOrCreate(x =>
|
||
(x.AltName == null || x.AltName.Equals(prof, StringComparison.OrdinalIgnoreCase)) &&
|
||
x.Name.Equals(altName, StringComparison.OrdinalIgnoreCase),
|
||
() => new Professor
|
||
{
|
||
AltName = prof,
|
||
Name = altName
|
||
});
|
||
|
||
professor.Add(profDb);
|
||
}
|
||
}
|
||
else
|
||
professor = null;
|
||
|
||
List<LectureHall>? hall = null;
|
||
List<Campus>? campuses;
|
||
if (groupInfo.Campuses != null && groupInfo.Campuses.Length != 0)
|
||
{
|
||
hall = [];
|
||
campuses = [];
|
||
for (var i = 0; i < groupInfo.Campuses.Length; i++)
|
||
{
|
||
var campus = groupInfo.Campuses[i];
|
||
campuses.Add(_campuses.GetOrCreate(
|
||
c => c.CodeName.Equals(campus, StringComparison.OrdinalIgnoreCase),
|
||
() => new Campus
|
||
{
|
||
CodeName = campus.ToUpper()
|
||
}));
|
||
if (groupInfo.LectureHalls == null || groupInfo.LectureHalls.Length <= i)
|
||
continue;
|
||
var lectureHall = groupInfo.LectureHalls[i];
|
||
|
||
hall.Add(_lectureHalls.GetOrCreate(l =>
|
||
l.Name.Equals(lectureHall, StringComparison.OrdinalIgnoreCase) &&
|
||
string.Equals(l.Campus?.CodeName, campuses[^1].CodeName, StringComparison.CurrentCultureIgnoreCase),
|
||
() => new LectureHall
|
||
{
|
||
Name = lectureHall,
|
||
Campus = campuses[^1]
|
||
}));
|
||
}
|
||
}
|
||
|
||
var discipline = _disciplines.GetOrCreate(
|
||
d => d.Name.Equals(groupInfo.Discipline, StringComparison.OrdinalIgnoreCase),
|
||
() => new Discipline
|
||
{
|
||
Name = groupInfo.Discipline
|
||
});
|
||
|
||
Lesson lesson = _lessons.GetOrCreate(l =>
|
||
l.IsEven == groupInfo.IsEven &&
|
||
l.DayOfWeek == groupInfo.Day &&
|
||
l.PairNumber == groupInfo.Pair &&
|
||
l.Discipline?.Name == discipline.Name &&
|
||
l.Group?.Name == group.Name,
|
||
() =>
|
||
{
|
||
var lesson = new Lesson
|
||
{
|
||
IsEven = groupInfo.IsEven,
|
||
DayOfWeek = groupInfo.Day,
|
||
PairNumber = groupInfo.Pair,
|
||
Discipline = discipline,
|
||
Group = group,
|
||
IsExcludedWeeks = groupInfo.IsExclude
|
||
};
|
||
|
||
if (groupInfo.SpecialWeek == null)
|
||
return lesson;
|
||
|
||
foreach (var week in groupInfo.SpecialWeek)
|
||
_specificWeeks.Create(() => new SpecificWeek
|
||
{
|
||
Lesson = lesson,
|
||
WeekNumber = week
|
||
});
|
||
|
||
return lesson;
|
||
});
|
||
|
||
var maxValue = int.Max(int.Max(professor?.Count ?? -1, hall?.Count ?? -1), 1);
|
||
|
||
for (var i = 0; i < maxValue; i++)
|
||
{
|
||
var prof = professor?.ElementAtOrDefault(i);
|
||
var lectureHall = hall?.ElementAtOrDefault(i);
|
||
_lessonAssociation.Create(() => new LessonAssociation
|
||
{
|
||
Professor = prof,
|
||
Lesson = lesson,
|
||
LectureHall = lectureHall,
|
||
TypeOfOccupation = typeOfOccupation
|
||
});
|
||
}
|
||
}
|
||
|
||
private async Task SaveChanges(CancellationToken cancellationToken)
|
||
{
|
||
foreach (var group in _groups.GetAll())
|
||
{
|
||
var existingGroup = await dbContext.Groups.FirstOrDefaultAsync(g => g.Id == group.Id, cancellationToken);
|
||
if (existingGroup != null)
|
||
dbContext.Remove(existingGroup);
|
||
}
|
||
|
||
await dbContext.Disciplines.BulkSynchronizeAsync(_disciplines.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
|
||
await dbContext.Professors.BulkSynchronizeAsync(_professors.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
|
||
await dbContext.TypeOfOccupations.BulkSynchronizeAsync(_typeOfOccupations.GetAll(), bulkOperation => bulkOperation.BatchSize = 100, cancellationToken);
|
||
await dbContext.Faculties.BulkSynchronizeAsync(_faculties.GetAll(), bulkOperation => bulkOperation.BatchSize = 100, cancellationToken);
|
||
await dbContext.Campuses.BulkSynchronizeAsync(_campuses.GetAll(), bulkOperation => bulkOperation.BatchSize = 10, cancellationToken);
|
||
await dbContext.LectureHalls.BulkSynchronizeAsync(_lectureHalls.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
|
||
await dbContext.Groups.BulkSynchronizeAsync(_groups.GetAll(), bulkOperation => bulkOperation.BatchSize = 100, cancellationToken);
|
||
await dbContext.Lessons.BulkSynchronizeAsync(_lessons.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
|
||
await dbContext.SpecificWeeks.BulkSynchronizeAsync(_specificWeeks.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
|
||
await dbContext.LessonAssociations.BulkSynchronizeAsync(_lessonAssociation.GetAll(), bulkOperation => bulkOperation.BatchSize = 1000, cancellationToken);
|
||
}
|
||
|
||
private async Task Sync(Func<CancellationToken, Task<List<GroupResult>>> parseDataAsync, CancellationToken cancellationToken)
|
||
{
|
||
Stopwatch watch = new();
|
||
watch.Start();
|
||
|
||
try
|
||
{
|
||
logger.LogDebug("Start parsing schedule");
|
||
var data = await parseDataAsync(cancellationToken);
|
||
|
||
watch.Stop();
|
||
var parsingTime = watch.ElapsedMilliseconds;
|
||
watch.Restart();
|
||
|
||
ParallelOptions options = new()
|
||
{
|
||
CancellationToken = cancellationToken,
|
||
MaxDegreeOfParallelism = Environment.ProcessorCount
|
||
};
|
||
|
||
logger.LogDebug("Start mapping parsed data");
|
||
Parallel.ForEach(data, options, ParallelSync);
|
||
|
||
watch.Stop();
|
||
var mappingTime = watch.ElapsedMilliseconds;
|
||
watch.Restart();
|
||
|
||
maintenanceMode.EnableMaintenanceMode();
|
||
|
||
logger.LogDebug("Start saving changing");
|
||
await SaveChanges(cancellationToken);
|
||
|
||
maintenanceMode.DisableMaintenanceMode();
|
||
|
||
watch.Stop();
|
||
|
||
logger.LogInformation("Parsing time: {ParsingTime}ms Mapping time: {MappingTime}ms Saving time: {SavingTime}ms Total time: {TotalTime}ms",
|
||
parsingTime, mappingTime, watch.ElapsedMilliseconds, parsingTime + mappingTime + watch.ElapsedMilliseconds);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
logger.LogError(ex, "An error occurred during synchronization.");
|
||
throw;
|
||
}
|
||
finally
|
||
{
|
||
maintenanceMode.DisableMaintenanceMode();
|
||
}
|
||
}
|
||
|
||
public async Task StartSync(CancellationToken cancellationToken)
|
||
{
|
||
var pairPeriods = config.Value.ScheduleSettings?.PairPeriod
|
||
.ToDictionary(x => x.Key, x => (x.Value.Start, x.Value.End));
|
||
|
||
var startTerm = config.Value.ScheduleSettings?.StartTerm;
|
||
|
||
if (pairPeriods == null || startTerm == null)
|
||
{
|
||
logger.LogWarning("It is not possible to synchronize the schedule due to the fact that the {Arg1} or {Arg2} variable is not initialized.",
|
||
nameof(pairPeriods),
|
||
nameof(startTerm));
|
||
|
||
return;
|
||
}
|
||
|
||
var parser = new Parser
|
||
{
|
||
Pairs = pairPeriods
|
||
.ToDictionary(x => x.Key,
|
||
x => (x.Value.Start, x.Value.End)),
|
||
TermStart = startTerm.Value.ToDateTime(new TimeOnly(0, 0, 0))
|
||
};
|
||
|
||
try
|
||
{
|
||
await Sync(parser.ParseAsync, cancellationToken);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
logger.LogError(ex, "An error occurred during synchronization.");
|
||
throw;
|
||
}
|
||
finally
|
||
{
|
||
maintenanceMode.DisableMaintenanceMode();
|
||
}
|
||
}
|
||
|
||
public async Task StartSync(List<(string File, string Campus)> files, CancellationToken cancellationToken)
|
||
{
|
||
await Task.Yield();
|
||
var pairPeriods = config.Value.ScheduleSettings?.PairPeriod
|
||
.ToDictionary(x => x.Key, x => (x.Value.Start, x.Value.End));
|
||
|
||
if (pairPeriods == null)
|
||
{
|
||
logger.LogWarning("It is not possible to synchronize the schedule due to the fact that the {Arg1} variable is not initialized.",
|
||
nameof(pairPeriods));
|
||
|
||
return;
|
||
}
|
||
|
||
try
|
||
{
|
||
Task<List<GroupResult>> ParseTask(CancellationToken ctx)
|
||
{
|
||
var mappedData = new ConcurrentBag<GroupResult>();
|
||
|
||
ParallelOptions options = new() { CancellationToken = ctx, MaxDegreeOfParallelism = Environment.ProcessorCount };
|
||
Parallel.ForEach(files, options, (file) =>
|
||
{
|
||
var parser = new Tools.Schedule.Parser.Parser();
|
||
var result = ConvertToGroupResults(parser.Parse(file.File, pairPeriods), file.Campus);
|
||
|
||
foreach (var item in result) mappedData.Add(item);
|
||
});
|
||
|
||
return Task.FromResult(mappedData.ToList());
|
||
}
|
||
|
||
await Sync(ParseTask, cancellationToken);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
logger.LogError(ex, "An error occurred during synchronization.");
|
||
throw;
|
||
}
|
||
finally
|
||
{
|
||
maintenanceMode.DisableMaintenanceMode();
|
||
}
|
||
}
|
||
|
||
private static List<GroupResult> ConvertToGroupResults(IEnumerable<GroupInfo> groups, string campusDefault, CancellationToken cancellationToken = default)
|
||
{
|
||
var result = new List<GroupResult>();
|
||
|
||
foreach (var group in groups)
|
||
{
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
|
||
foreach (var day in group.Days)
|
||
{
|
||
foreach (var pair in day.Lessons)
|
||
{
|
||
foreach (var lesson in pair.Value)
|
||
{
|
||
if (string.IsNullOrWhiteSpace(lesson.TypeOfOccupation))
|
||
continue;
|
||
|
||
var (weeks, isExclude) = ParseWeeks(lesson.Discipline);
|
||
|
||
var (lectureHalls, campuses) = ParseLectureHalls(lesson.LectureHall, campusDefault);
|
||
|
||
var groupResult = new GroupResult
|
||
{
|
||
Day = day.DayOfWeek,
|
||
Pair = pair.Key,
|
||
IsEven = lesson.IsEven,
|
||
Group = group.GroupName,
|
||
Discipline = NormalizeDiscipline(lesson.Discipline),
|
||
Professor = ParseProfessors(lesson.Professor),
|
||
TypeOfOccupation = lesson.TypeOfOccupation,
|
||
LectureHalls = lectureHalls,
|
||
Campuses = campuses,
|
||
SpecialWeek = weeks,
|
||
IsExclude = isExclude
|
||
};
|
||
|
||
result.Add(groupResult);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
return result;
|
||
}
|
||
|
||
private static string[]? ParseProfessors(string? input)
|
||
{
|
||
if (string.IsNullOrWhiteSpace(input)) return null;
|
||
|
||
var normalized = Regex.Replace(input
|
||
.Replace("\n", " ")
|
||
.Replace(",", " "),
|
||
@"\s+", " ").Trim();
|
||
|
||
return ProfessorFullName().Matches(normalized)
|
||
.Select(m => $"{m.Groups["surname"].Value} {m.Groups["initials"].Value}".Trim())
|
||
.Where(x => !string.IsNullOrEmpty(x))
|
||
.ToArray();
|
||
}
|
||
|
||
private static (int[]? weeks, bool? isExclude) ParseWeeks(string discipline)
|
||
{
|
||
var match = ParseSpecificWeeks().Match(discipline);
|
||
|
||
if (!match.Success) return (null, null);
|
||
|
||
var numbers = new List<int>();
|
||
var ranges = match.Groups[2].Value.Split(',');
|
||
|
||
foreach (var range in ranges)
|
||
{
|
||
if (range.Contains('-'))
|
||
{
|
||
var parts = range.Split('-');
|
||
if (int.TryParse(parts[0], out var start) &&
|
||
int.TryParse(parts[1], out var end))
|
||
{
|
||
numbers.AddRange(Enumerable.Range(start, end - start + 1));
|
||
}
|
||
}
|
||
else
|
||
if (int.TryParse(range, out var num)) numbers.Add(num);
|
||
}
|
||
|
||
return (
|
||
weeks: numbers.Distinct().OrderBy(x => x).ToArray(),
|
||
isExclude: match.Groups[1].Success
|
||
);
|
||
}
|
||
|
||
private static string NormalizeDiscipline(string input)
|
||
{
|
||
var normalized = Regex.Replace(input
|
||
.Replace("\n", " ")
|
||
.Replace("\r", " "),
|
||
@"\s{2,}", " ");
|
||
|
||
normalized = Regex.Replace(normalized,
|
||
@"(\S+)\s(\S{3,})",
|
||
"$1 $2");
|
||
|
||
normalized = ParseSpecificWeeks().Replace(normalized, "");
|
||
|
||
return normalized.Trim();
|
||
}
|
||
|
||
private static (string[]? lectureHalls, string[]? campuses) ParseLectureHalls(string? input, string defaultCampus)
|
||
{
|
||
if (string.IsNullOrWhiteSpace(input))
|
||
return (null, null);
|
||
|
||
var matches = ParseLectureCampus().Matches(input);
|
||
var lectureHalls = new List<string>();
|
||
var campuses = new List<string>();
|
||
|
||
foreach (Match match in matches)
|
||
{
|
||
if (match.Groups["lectureWithCampus"].Success)
|
||
{
|
||
var raw = match.Value.Split('(');
|
||
var campus = raw.LastOrDefault()?.Trim(')').Trim();
|
||
var lecture = raw.FirstOrDefault()?.Trim();
|
||
|
||
if (string.IsNullOrEmpty(campus) || string.IsNullOrEmpty(lecture))
|
||
continue;
|
||
|
||
campuses.Add(campus);
|
||
lectureHalls.Add(lecture);
|
||
}
|
||
else if (match.Groups["lecture"].Success)
|
||
{
|
||
var lecture = match.Value.Trim();
|
||
if (string.IsNullOrEmpty(lecture))
|
||
continue;
|
||
|
||
campuses.Add(defaultCampus);
|
||
lectureHalls.Add(lecture);
|
||
}
|
||
}
|
||
|
||
return (
|
||
lectureHalls: lectureHalls.ToArray(),
|
||
campuses: campuses.ToArray()
|
||
);
|
||
}
|
||
|
||
|
||
|
||
[GeneratedRegex(@"\w{4}-\d{2}-\d{2}(?=\s?\d?\s?[Пп]/?[Гг]\s?\d?)?")]
|
||
private static partial Regex OnlyGroupName();
|
||
|
||
[GeneratedRegex(@"(?<surname>[А-ЯЁ][а-яё]+(-[А-ЯЁ][а-яё]+)?)\s*(?<initials>[А-ЯЁ]\.[А-ЯЁ]?\.?)?", RegexOptions.IgnorePatternWhitespace)]
|
||
private static partial Regex ProfessorFullName();
|
||
|
||
[GeneratedRegex(@"([Кк]р\.?)?\s*((\d+-\d+|\d+)(,\s*\d+(-\d+)?)*)\s*[Нн]\.?", RegexOptions.IgnoreCase, "ru-RU")]
|
||
private static partial Regex ParseSpecificWeeks();
|
||
|
||
[GeneratedRegex(@"(?<lectureWithCampus>[^,.\n]+\s?\([А-Яа-яA-Za-z]+-?\d+\))|(?<lecture>[^,.\n]+)")]
|
||
private static partial Regex ParseLectureCampus();
|
||
} |