Merge branch 'master' into users/devi/jobs-backend

Author: devidnyk

.gitignore

@@ -197,4 +197,5 @@ dependency-reduced-pom.xml
 README.html
 *.iml
 .idea
-.exercism
+.exercism
+/src/.vs

src/Backend/Backend.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
+<Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
@@ -13,14 +13,18 @@
     <PackageReference Include="Azure.Core" Version="1.49.0" />
     <PackageReference Include="Azure.Identity" Version="1.17.0" />
     <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.23.0" />
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.53.1" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.54.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.ApplicationInsights" Version="2.23.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="9.0.6" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Common\Common.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="Docs\" />
+  </ItemGroup>
+
 </Project>

src/Backend/Backend.csproj.user

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <ActiveDebugProfile>https</ActiveDebugProfile>
+  </PropertyGroup>
+</Project>

src/Backend/Controllers/ProblemsController.cs

@@ -1,21 +1,24 @@
 namespace Backend.Controllers
 {
+    using Backend.Filters;
     using Backend.Models.Public;
     using Backend.Operations;
+    using Common.Models;
     using Microsoft.AspNetCore.Mvc;
-    using ProblemPublicModel = Common.Models.Problem;
 
     [ApiController]
     [Route("api")]
     public class ProblemsController : ControllerBase
     {
-        private AppContext appContext;
         private readonly ILogger<ProblemsController> logger;
         private readonly IConfiguration configuration;
-        public ProblemsController(AppContext appContext, ILogger<ProblemsController> logger, IConfiguration configuration)
+        private readonly DataProvider dataProvider;
+        public ProblemsController(ILogger<ProblemsController> logger,
+            DataProvider dataProvider,
+            IConfiguration configuration)
         {
-            this.appContext = appContext;
             this.logger = logger;
+            this.dataProvider = dataProvider;
             this.configuration = configuration;
         }
 
@@ -28,23 +31,23 @@ public ActionResult<string> GetHome()
 
         [HttpGet]
         [Route("problems")]
-        public async Task<ActionResult<IEnumerable<ProblemPublicModel>>> GetProblems(
+        public async Task<ActionResult<IEnumerable<Problem>>> GetProblems(
             [FromQuery(Name = QueryParam.Skip)] int skip = 0,
             [FromQuery(Name = QueryParam.Limit)] int limit = 50,
             [FromQuery(Name = QueryParam.Company)] List<string>? companies = null,
-            [FromQuery(Name = QueryParam.Difficulty)] List<Common.Models.Difficulty>? difficulties = null,
+            [FromQuery(Name = QueryParam.Difficulty)] List<Difficulty>? difficulties = null,
             [FromQuery(Name = QueryParam.Tag)] List<string>? tags = null)
         {
             var filter = new ProblemFilter(skip, limit, companies, difficulties, tags);
-            var filteredProblems = await appContext.dataProvider.GetProblemsAsync(filter);
+            var filteredProblems = await dataProvider.GetProblemsAsync(filter);
             return Ok(filteredProblems);
         }
 
         [HttpGet]
         [Route("problems/{id}")]
-        public async Task<ActionResult<ProblemPublicModel>> GetProblems(string id)
+        public async Task<ActionResult<Problem>> GetProblems(string id)
         {
-            var problem = await appContext.dataProvider.GetProblemByIdAsync(id);
+            var problem = await dataProvider.GetProblemByIdAsync(id);
             if (problem != null)
             {
                 return Ok(problem);

src/Backend/Docs/NextPlanOfAction.md

@@ -0,0 +1,8 @@
+## Next Plans
+1. **Create functions for async updates for Jobs backend**
+2. **Integrate Redis for fast retreival**
+3. **Notification System upon new Job post arrival**
+	3.1. **Will require integration to some async framework.**
+	3.2. **Using Servicebus can be costly might use some other message broker**	
+4. **Custom tracing framework using Prometheus and Grafana (learning purpose)**
+5. **Make the system scalable enough to register a new scraping source easily**

src/Backend/Filters/IFilter.cs

@@ -0,0 +1,9 @@
+using Common.Models;
+
+namespace Backend.Filters
+{
+    public interface IFilter
+    {
+        public List<Problem> ApplyFilterAsync(List<Problem> problems);
+    }    
+}

src/Backend/Operations/ProblemFilter.cs src/Backend/Filters/ProblemFilter.cssrc/Backend/Operations/ProblemFilter.cs renamed to src/Backend/Filters/ProblemFilter.cs

@@ -1,38 +1,41 @@
-namespace Backend.Operations
+using Common.Models;
+
+namespace Backend.Filters
 {
     public class ProblemFilter : IFilter
     {
         private int skip = 0;
         private int limit = 50;
         private List<string> companies;
-        private List<Common.Models.Difficulty> difficulties;
+        private List<Difficulty> difficulties;
         private List<string> tags;
 
-        public ProblemFilter(int skip, int limit, List<string>? companies, List<Common.Models.Difficulty>? difficulties, List<string>? tags)
+        public ProblemFilter(int skip, int limit, List<string>? companies, List<Difficulty>? difficulties, List<string>? tags)
         {
             this.skip = skip;
             this.limit = Math.Min(limit, 50);
             this.companies = companies ?? new List<string>();
-            this.difficulties = difficulties ?? new List<Common.Models.Difficulty>();
+            this.difficulties = difficulties ?? new List<Difficulty>();
             this.tags = tags ?? new List<string>();
         }
 
-        public List<Common.Models.Problem> ApplyFilterAsync(List<Common.Models.Problem> problems)
+        public List<Problem> ApplyFilterAsync(List<Problem> problems)
         {
-            List<Common.Models.Problem> filteredProblems = problems;
+            List<Problem> filteredProblems = problems;
 
             // TODO: Add tags filtering logic with company 
-            if (companies != null && companies.Count > 0)
+            if (companies != null && companies.Count > 0 || tags != null && tags.Count > 0)
             {
                 filteredProblems = filteredProblems.Where(
                     p => p.companies != null &&
-                    p.companies.Keys.Any(
-                        c => this.companies.Contains(c, StringComparer.OrdinalIgnoreCase))).ToList();
+                    p.companies.Any(kv =>
+                        (companies == null || companies.Count== 0 || companies.Contains(kv.Key, StringComparer.OrdinalIgnoreCase)) &&
+                        kv.Value.Any(t => tags == null || tags.Count == 0 || tags.Contains(t, StringComparer.OrdinalIgnoreCase)))).ToList();
             }
 
             if (difficulties != null && difficulties.Count > 0)
             {
-                filteredProblems = filteredProblems.Where(p => this.difficulties.Contains(p.difficulty)).ToList();
+                filteredProblems = filteredProblems.Where(p => difficulties.Contains(p.difficulty)).ToList();
             }
 
             filteredProblems = filteredProblems.Skip(skip).Take(limit).ToList();

src/Backend/Models/Internal/Problem.cs

@@ -1,24 +1,18 @@
 namespace Backend.Operations
 {
+    using Backend.Filters;
+    using Common.Cache;
+    using Common.Constants;
     using Common.Models;
-    using Microsoft.Azure.Cosmos;
 
     public class DataProvider
     {
-        private const int RefreshIntervalInHours = 3;
-        private CosmosClient cosmosClient;
-        private readonly IConfiguration configuration;
-        ILogger<DataProvider> logger;
-        private DateTime lastLoadedTime = DateTime.MinValue;
-        private Task backgroundRefreshTask;
-        private Dictionary<string, Problem> problemsCache = new Dictionary<string, Problem>(StringComparer.OrdinalIgnoreCase);
-
-        public DataProvider(CosmosClient client, IConfiguration configuration, ILogger<DataProvider> logger)
+        private ICache _problemCache;
+        private ILogger<DataProvider> _logger;
+        public DataProvider([FromKeyedServices(CacheConstants.ProblemCacheKey)] ICache problemCache, ILogger<DataProvider> logger)
         {
-            this.cosmosClient = client;
-            this.configuration = configuration;
-            this.logger = logger;
-            this.backgroundRefreshTask = Task.Run(() => this.StartBackgroundRefreshAsync(CancellationToken.None));
+            _problemCache = problemCache;
+            _logger = logger;
         }
 
         public async Task<List<Problem>> GetProblemsAsync(IFilter? filter = null)
@@ -43,68 +37,17 @@ public async Task<List<Problem>> GetProblemsAsync(IFilter? filter = null)
 
         private async Task<Dictionary<string, Problem>> GetAllProblemsAsync()
         {
-            if (problemsCache.Count == 0)
+            if (_problemCache.Contains(CacheConstants.ProblemCacheKey))
             {
-                await LoadLatestDataAsync();
-                if (problemsCache.Count == 0)
-                {
-                    this.logger.LogWarning("No problems found in the cache after loading data.");
-                }
+                _logger.LogInformation("Problem cache hit. Retrieving data from cache.");
             }
-            return problemsCache;
-        }
-
-        private async Task LoadLatestDataAsync()
-        {
-            int maxRetries = 3;
-            for(int i = 0; i < maxRetries; i++)
+            else
             {
-                try
-                {
-                    var dbId = configuration.GetValue<string>("ApplicationSettings:CosmosDbDatabaseId");
-                    var containerId = configuration.GetValue<string>("ApplicationSettings:CosmosDbContainerId");
-                    var db = cosmosClient.GetDatabase(dbId);
-                    var container = db.GetContainer(containerId);
-
-                    var query = "SELECT * FROM c";
-                    var queryDefinition = new QueryDefinition(query);
-                    var queryResultSetIterator = container.GetItemQueryIterator<ProblemSchema>(queryDefinition);
-
-                    List<Problem> results = new List<Problem>();
-                    while (queryResultSetIterator.HasMoreResults)
-                    {
-                        var response = await queryResultSetIterator.ReadNextAsync();
-                        results.AddRange(response.Select(item => new Problem(item)));
-                    }
-
-                    lastLoadedTime = DateTime.UtcNow;
-                    problemsCache = results.ToDictionary(p => p.id, StringComparer.OrdinalIgnoreCase);
-                    this.logger.LogInformation($"Loaded {problemsCache.Count} problems from Cosmos DB at {lastLoadedTime}");
-                    break;
-                }
-                catch (Exception ex)
-                {
-                    this.logger.LogError($"Error loading data from Cosmos DB. {ex}");
-                    await Task.Delay(TimeSpan.FromSeconds(2 * (i + 1)));
-                }
-            }
-        }
-        
-        public async Task StartBackgroundRefreshAsync(CancellationToken cancellationToken)
-        {
-            while (!cancellationToken.IsCancellationRequested)
-            {
-                try
-                {
-                    await LoadLatestDataAsync();
-                }
-                catch (Exception ex)
-                {
-                    this.logger.LogError($"Error during background data refresh: {ex}");
-                }
-
-                await Task.Delay(TimeSpan.FromHours(RefreshIntervalInHours), cancellationToken);
+                _logger.LogInformation("Problem cache miss. Loading data into cache.");
+                await _problemCache.Populate();
             }
+            
+            return _problemCache.Get<Dictionary<string, Problem>>(CacheConstants.ProblemCacheKey) ?? new Dictionary<string, Problem>(StringComparer.OrdinalIgnoreCase);
         }
     }
 }