From fee588a67f0fad1ab77d2f07ec7a37c30ed121ad Mon Sep 17 00:00:00 2001 From: Alex Mickelson Date: Mon, 1 Jan 2024 20:45:33 -0700 Subject: [PATCH] got tracing and loggin working in opentelemetry --- Management.Web/Management.Web.csproj | 4 ++ Management.Web/Pages/Index.razor | 6 +- Management.Web/Program.cs | 49 +++++++++++++-- Management.Web/appsettings.Development.json | 4 +- Management.Web/appsettings.json | 2 +- .../Features/Configuration/CoursePlanner.cs | 17 +++++- .../Services/Files/FileStorageManager.cs | 20 ++++-- .../Services/Files/LoadMarkdownCourse.cs | 2 +- .../Services/Files/SaveMarkdownCourse.cs | 24 +++----- Management/Services/MyLogger.cs | 14 ++--- docker-compose.yml | 61 ++++++++++++++----- ops/otel-collector-config.yml | 35 +++++++++++ 12 files changed, 181 insertions(+), 57 deletions(-) create mode 100644 ops/otel-collector-config.yml diff --git a/Management.Web/Management.Web.csproj b/Management.Web/Management.Web.csproj index f25017d..cfd2ba2 100644 --- a/Management.Web/Management.Web.csproj +++ b/Management.Web/Management.Web.csproj @@ -8,6 +8,10 @@ + + + + diff --git a/Management.Web/Pages/Index.razor b/Management.Web/Pages/Index.razor index da84890..08f949d 100644 --- a/Management.Web/Pages/Index.razor +++ b/Management.Web/Pages/Index.razor @@ -1,7 +1,5 @@ @page "/" @using CanvasModel.EnrollmentTerms -@using Management.Web.Shared.Course -@using Management.Web.Shared.Semester @using CanvasModel.Courses @using Microsoft.AspNetCore.Components.Server.ProtectedBrowserStorage @using LocalModels @@ -47,7 +45,7 @@ - + @if(!showNewFile) {
@@ -79,4 +77,4 @@
-@* *@ \ No newline at end of file +@* *@ diff --git a/Management.Web/Program.cs b/Management.Web/Program.cs index aa1e23f..674bc6a 100644 --- a/Management.Web/Program.cs +++ b/Management.Web/Program.cs @@ -10,13 +10,14 @@ global using LocalModels; global using Management.Planner; global using Management.Web.Shared.Components; global using Management.Web.Shared; -global using Management.Web.Shared.Components.Forms; -using Microsoft.AspNetCore.Components; -using Microsoft.AspNetCore.Components.Web; using dotenv.net; using Microsoft.AspNetCore.Hosting.Server; using Microsoft.AspNetCore.Hosting.Server.Features; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; DotEnv.Load(); @@ -32,6 +33,42 @@ if (canvas_url == null) Environment.SetEnvironmentVariable("CANVAS_URL", "https://snow.instructure.com"); } +const string serviceName = "canvas-management"; + +// builder.Logging.AddOpenTelemetry(options => +// { +// options +// .SetResourceBuilder( +// ResourceBuilder +// .CreateDefault() +// .AddService(serviceName) +// ) +// .AddOtlpExporter(o => +// { +// o.Endpoint = new Uri("http://localhost:4317/"); +// }) +// .AddConsoleExporter(); +// }); + +builder.Services.AddOpenTelemetry() + .ConfigureResource(resource => resource.AddService(serviceName)) + .WithTracing(tracing => tracing + .AddSource(DiagnosticsConfig.SourceName) + .AddOtlpExporter(o => + { + o.Endpoint = new Uri("http://localhost:4317/"); + }) + .AddAspNetCoreInstrumentation() + .AddConsoleExporter() + ); +// .WithMetrics(metrics => metrics +// .AddOtlpExporter(o => { +// o.Endpoint = new Uri("http://localhost:4317/"); +// }) +// .AddAspNetCoreInstrumentation() +// .AddConsoleExporter() +// ); + // Add services to the container. builder.Services.AddRazorPages(); builder.Services.AddServerSideBlazor(); @@ -59,7 +96,7 @@ builder.Services.AddScoped(); builder.Services.AddSignalR(e => { - e.MaximumReceiveMessageSize = 102400000; + e.MaximumReceiveMessageSize = 102400000; }); @@ -90,7 +127,7 @@ var addresses = app.Services.GetService()?.Features.Get logger; private readonly FileStorageManager fileStorageManager; private readonly CanvasService canvas; + private readonly ILogger _otherLogger; + public bool LoadingCanvasData { get; internal set; } = false; - public CoursePlanner(MyLogger logger, FileStorageManager fileStorageManager, CanvasService canvas) + public CoursePlanner( + MyLogger logger, + FileStorageManager fileStorageManager, + CanvasService canvas, + ILogger otherLogger + ) { this.logger = logger; this.fileStorageManager = fileStorageManager; this.canvas = canvas; + _otherLogger = otherLogger; + _otherLogger.LogInformation("testing other logging"); } private Timer? _debounceTimer; @@ -34,7 +43,7 @@ public class CoursePlanner get => _localCourse; set { - + using var activity = DiagnosticsConfig.Source?.StartActivity("Loading Course"); if (value == null) { _localCourse = null; @@ -110,6 +119,8 @@ public class CoursePlanner IEnumerable canvasAssignmentGroups )> LoadCanvasData() { + + using var activity = DiagnosticsConfig.Source.StartActivity("Loading Canvas Data to Course Planner"); LoadingCanvasData = true; StateHasChanged?.Invoke(); @@ -172,7 +183,7 @@ public class CoursePlanner ? g : g with {CanvasId = canvasGroup.Id}; }) - } + } }; } } diff --git a/Management/Services/Files/FileStorageManager.cs b/Management/Services/Files/FileStorageManager.cs index 81e116a..0e9bba0 100644 --- a/Management/Services/Files/FileStorageManager.cs +++ b/Management/Services/Files/FileStorageManager.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using LocalModels; using Management.Services; using YamlDotNet.Serialization; @@ -7,21 +8,24 @@ public class FileStorageManager private readonly MyLogger logger; private readonly CourseMarkdownLoader _courseMarkdownLoader; private readonly MarkdownCourseSaver _saveMarkdownCourse; + private readonly ILogger _otherLogger; private readonly string _basePath; public FileStorageManager( MyLogger logger, CourseMarkdownLoader courseMarkdownLoader, - MarkdownCourseSaver saveMarkdownCourse + MarkdownCourseSaver saveMarkdownCourse, + ILogger otherLogger ) { + using var activity = DiagnosticsConfig.Source.StartActivity("loading storage directory"); this.logger = logger; _courseMarkdownLoader = courseMarkdownLoader; _saveMarkdownCourse = saveMarkdownCourse; + _otherLogger = otherLogger; _basePath = FileConfiguration.GetBasePath(); this.logger.Log("Using storage directory: " + _basePath); - } public async Task SaveCourseAsync(LocalCourse course, LocalCourse? previouslyStoredCourse) { @@ -36,6 +40,7 @@ public class FileStorageManager public async Task> LoadSavedMarkdownCourses() { + return await _courseMarkdownLoader.LoadSavedMarkdownCourses(); } @@ -43,11 +48,18 @@ public class FileStorageManager { if(!Directory.Exists(_basePath)) throw new DirectoryNotFoundException($"Cannot get empty directories, {_basePath} does not exist"); - + return Directory .GetDirectories(_basePath, "*") .Where(dir => !Directory.EnumerateFileSystemEntries(dir).Any()) .ToArray(); } -} \ No newline at end of file +} + + +public static class DiagnosticsConfig +{ + public const string SourceName = "canvas-management-source"; + public static ActivitySource Source = new ActivitySource(SourceName); +} diff --git a/Management/Services/Files/LoadMarkdownCourse.cs b/Management/Services/Files/LoadMarkdownCourse.cs index db70da7..b21b4da 100644 --- a/Management/Services/Files/LoadMarkdownCourse.cs +++ b/Management/Services/Files/LoadMarkdownCourse.cs @@ -128,4 +128,4 @@ public class CourseMarkdownLoader return await Task.WhenAll(quizPromises); } -} \ No newline at end of file +} diff --git a/Management/Services/Files/SaveMarkdownCourse.cs b/Management/Services/Files/SaveMarkdownCourse.cs index c9a255c..4d79d67 100644 --- a/Management/Services/Files/SaveMarkdownCourse.cs +++ b/Management/Services/Files/SaveMarkdownCourse.cs @@ -1,17 +1,11 @@ using System.Threading.Tasks.Sources; using LocalModels; -using Management.Services; +namespace Management.Services; -public class MarkdownCourseSaver +public class MarkdownCourseSaver(MyLogger logger) { - private readonly MyLogger logger; - private readonly string _basePath; - - public MarkdownCourseSaver(MyLogger logger) - { - this.logger = logger; - _basePath = FileConfiguration.GetBasePath(); - } + private readonly MyLogger _logger = logger; + private readonly string _basePath = FileConfiguration.GetBasePath(); public async Task Save(LocalCourse course, LocalCourse? previouslyStoredCourse) { @@ -70,7 +64,7 @@ public class MarkdownCourseSaver { var markdownPath = quizzesDirectory + "/" + quiz.Name + ".md"; ; var quizMarkdown = quiz.ToMarkdown(); - logger.Log("saving quiz " + markdownPath); + _logger.Log("saving quiz " + markdownPath); await File.WriteAllTextAsync(markdownPath, quizMarkdown); } } @@ -94,7 +88,7 @@ public class MarkdownCourseSaver foreach (var file in filesToDelete) { - logger.Log($"removing old quiz, it has probably been renamed {file}"); + _logger.Log($"removing old quiz, it has probably been renamed {file}"); File.Delete(file); } @@ -117,7 +111,7 @@ public class MarkdownCourseSaver var assignmentMarkdown = assignment.ToMarkdown(); var filePath = assignmentsDirectory + "/" + assignment.Name + ".md"; - logger.Log("saving assignment " + filePath); + _logger.Log("saving assignment " + filePath); await File.WriteAllTextAsync(filePath, assignmentMarkdown); } } @@ -141,9 +135,9 @@ public class MarkdownCourseSaver foreach (var file in filesToDelete) { - logger.Log($"removing old assignment, it has probably been renamed {file}"); + _logger.Log($"removing old assignment, it has probably been renamed {file}"); File.Delete(file); } } -} \ No newline at end of file +} diff --git a/Management/Services/MyLogger.cs b/Management/Services/MyLogger.cs index 6bb22b5..13364b5 100644 --- a/Management/Services/MyLogger.cs +++ b/Management/Services/MyLogger.cs @@ -5,22 +5,22 @@ namespace Management.Services; public class MyLogger { - private readonly ILogger baseLogger; + private readonly ILogger _baseLogger; public MyLogger(ILogger baseLogger) { - this.baseLogger = baseLogger; + this._baseLogger = baseLogger; } public void Log( string message, - LogLevel logLevel = LogLevel.Information, + // LogLevel logLevel = LogLevel.Information, [CallerMemberName] string memberName = "" ) { var finalMessage = $"[{typeof(T)}.{memberName}] {message}"; - baseLogger.Log(logLevel, finalMessage); + _baseLogger.LogInformation(finalMessage); Console.WriteLine(finalMessage); } @@ -32,7 +32,7 @@ public class MyLogger { var finalMessage = $"[{typeof(T)}.{memberName}] {message}"; - baseLogger.Log(logLevel, finalMessage); + _baseLogger.Log(logLevel, finalMessage); Console.WriteLine(finalMessage); } public void Error( @@ -43,7 +43,7 @@ public class MyLogger { var finalMessage = $"ERROR: [{typeof(T)}.{memberName}] {message}"; - baseLogger.Log(logLevel, finalMessage); + _baseLogger.Log(logLevel, finalMessage); Console.WriteLine(finalMessage); } -} \ No newline at end of file +} diff --git a/docker-compose.yml b/docker-compose.yml index ba0a592..a64f74a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,49 @@ services: - canvas_manager: - image: alexmickelson/canvas_management:1 - user: "1000:1000" - ports: - - 8080:8080 - env_file: - - .env - environment: - - storageDirectory=/app/storage + # canvas_manager: + # image: alexmickelson/canvas_management:1 + # user: "1000:1000" + # ports: + # - 8080:8080 + # env_file: + # - .env + # environment: + # - storageDirectory=/app/storage + # volumes: + # # - ./storage:/app/storage + # # - ~/projects/faculty/1410/2023-fall-alex/modules:/app/storage/1410 + # - ~/projects/faculty/1810/2024-spring-alex/modules:/app/storage/web_intro + # - ~/projects/faculty/1400/2024_spring_alex/modules:/app/storage/1400 + # - ~/projects/faculty/1405/2024_spring_alex/modules:/app/storage/1405 + + collector: + image: otel/opentelemetry-collector-contrib volumes: - # - ./storage:/app/storage - # - ~/projects/faculty/1410/2023-fall-alex/modules:/app/storage/1410 - - ~/projects/faculty/1810/2024-spring-alex/modules:/app/storage/web_intro - - ~/projects/faculty/1400/2024_spring_alex/modules:/app/storage/1400 - - ~/projects/faculty/1405/2024_spring_alex/modules:/app/storage/1405 \ No newline at end of file + - ./ops/otel-collector-config.yml:/etc/otelcol-contrib/config.yaml + ports: + - 1888:1888 # pprof extension + - 8888:8888 # Prometheus metrics exposed by the Collector + - 8889:8889 # Prometheus exporter metrics + - 13133:13133 # health_check extension + - 4317:4317 # OTLP gRPC receiver + - 4318:4318 # OTLP http receiver + - 55679:55679 # zpages extension + + # The zipkin process services the UI, and also exposes a POST endpoint that + # instrumentation can send trace data to. + zipkin: + image: ghcr.io/openzipkin/zipkin-slim + container_name: zipkin + # Environment settings are defined here https://github.com/openzipkin/zipkin/blob/master/zipkin-server/README.md#environment-variables + environment: + - STORAGE_TYPE=mem + # Point the zipkin at the storage backend + # - MYSQL_HOST=mysql + # Uncomment to enable self-tracing + # - SELF_TRACING_ENABLED=true + # Uncomment to increase heap size + # - JAVA_OPTS=-Xms128m -Xmx128m -XX:+ExitOnOutOfMemoryError + ports: + # Port used for the Zipkin UI and HTTP Api + - 9411:9411 + # Uncomment to enable debug logging + # command: --logging.level.zipkin2=DEBUG diff --git a/ops/otel-collector-config.yml b/ops/otel-collector-config.yml new file mode 100644 index 0000000..54fcb2c --- /dev/null +++ b/ops/otel-collector-config.yml @@ -0,0 +1,35 @@ +receivers: + otlp: + protocols: + grpc: + http: + +processors: + batch: + +exporters: + # otlp: + # endpoint: otelcol:4317 + zipkin: + endpoint: http://zipkin:9411/api/v2/spans + +extensions: + health_check: + pprof: + zpages: + +service: + extensions: [health_check, pprof, zpages] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [zipkin] + # metrics: + # receivers: [] + # processors: [batch] + # exporters: [] + # logs: + # receivers: [] + # processors: [batch] + # exporters: []