llm api working checkpoint

Reviewed-on: #4

README.md

@@ -1,4 +1,9 @@
-## agologum
+
-A web server/web client template.
+# agologum
-Backend: a .NET web API with a [READACTED] database (probably prostgressql for tinkering)
+
-Frontend: Vue.js because I enjoy life and the splendor of God's creation
+Agologum is a web app that I'm playing around with for no particular purpose or goal other than playing around like a kid building legos.
+- Frontend: Vue.js because I enjoy life and the splendor of God's creation
+- Backend: .NET Web Api despite what Microsoft did to my family
+- Database: Postgres because its the postbest
+
+Implore scripts/DEV_README.md if you're looking to build and test in your own environment. Most of the testing I'm doing is live because I fear only one man (not myself.)

api/Program.cs

@@ -29,6 +29,8 @@ builder.Services.AddScoped<UserService>();
 builder.Services.AddScoped<ItemService>();
 builder.Services.AddScoped<JwtService>();
 // if this grows sufficiently large we can put elsewhere
+// misc services that I didnt make
+builder.Services.AddHttpClient();
 
 // configuration for jwt authentication
 builder.Services.AddIdentity<User, IdentityRole>()
@@ -110,7 +112,7 @@ if (app.Environment.IsEnvironment("Development")) {
     app.UseHttpsRedirection();
 }
 
-app.MapControllers();
+app.MapControllers(); // automatically scours src/Controllers for controller classes
 
 // attempt enitity-framework migrations at startup. love you stack overflow
 using (var scope = app.Services.CreateScope()) {
@@ -125,7 +127,6 @@ using (var scope = app.Services.CreateScope()) {
             Thread.Sleep(5000);
         }
     }
-
 }
 
 app.Run();

api/docker-compose.prod.yaml

@@ -9,10 +9,14 @@ services:
     environment:
       ConnectionStrings__DefaultConnection: Host=agologum-db;Port=5432;Database=agologum;Username=agologum;Password=${POSTGRES_PASSWORD}
       Jwt__Key: ${JWT_SECRET} # must export the secret as a variable in the ci script
+      # a better option is to seed a .env file in the ci script from the gitea secrets
+      # then import that file into the container
     ports:
       - "5000:5000"
     networks:
       - agologum-net
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
 
 networks:
   agologum-net:

api/src/Controllers/ItemsController.cs

@@ -6,9 +6,11 @@ using agologumApi.Models;
 using agologumApi.Services;
 
 [ApiController]
-[Route("api/[controller]")]
+[Route("api/[controller]")] // generally good practice to make this explicit
 public class ItemsController : ControllerBase {
 
+    // TODO: (global) controller's might eventually need more services so its gonna be a good idea to give them more meaningful names
+    // i.e. just name it itemService_
     private readonly ItemService service_;
 
     public ItemsController(ItemService service) {

api/src/Controllers/LlmController.cs

@@ -0,0 +1,89 @@
+
+// system usings
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Authorization;
+using System.Net.Http;
+using System.Text;
+using System.Text.Json;
+// agologum usings
+using agologumApi.Models;
+//using agologumApi.Services;
+
+[ApiController]
+[Route("api/chat")]
+public class LlmController : ControllerBase {
+
+    // eventually this will be where most of the app sits
+    // might create src/Services/Llm/... to better organize
+    // private readonly LlmService llmService_;
+    private readonly IHttpClientFactory httpClientFactory_;
+    private readonly string LLAMA_URL = "http://host.docker.internal:8010/completion"; // TODO: can just put this in appsettings
+
+    public LlmController(/* LlmService llmService, */ IHttpClientFactory httpClientFactory) {
+        // llmService_ = llmService;
+        httpClientFactory_ = httpClientFactory;
+    }
+
+    // posting to api/chat/stream with a prompt in the body will return an sse (server sent events) stream of the llm text
+    // [Authorize(Policy = Permission.SensitiveData_Read)] // will make secret after testing
+    [HttpPost("stream")]
+    public async Task streamPrompt([FromBody] PromptRequest prompt) {
+        
+        var httpClient = httpClientFactory_.CreateClient(); // recreating this on every prompt request? idk seems 🐟y
+
+        // json object for llama to parse
+        var llamaRequest = new {
+            prompt = prompt.Prompt, // alternative would be a messages array
+            stream = true, // real time token streaming
+            n_predict = 256 // max amount of tokens
+            // other options are temperature, top_p and top_k for vocabulary diversity, stop for stop tokens
+            // https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md
+            // llm loras lol
+        };
+
+        // http request to send to the llama server
+        var httpRequest = new HttpRequestMessage(HttpMethod.Post, LLAMA_URL) {
+            Content = new StringContent(JsonSerializer.Serialize(llamaRequest), Encoding.UTF8, "application/json")
+        };
+
+        // send request
+        var response = await httpClient.SendAsync(httpRequest, HttpCompletionOption.ResponseHeadersRead, HttpContext.RequestAborted);
+        response.EnsureSuccessStatusCode();
+        Response.StatusCode = 200;
+        Response.ContentType = "text/plain"; // because its async dotnet lets us do this monstrosity
+
+        // now to handle the response stream
+        await using var responseStream = await response.Content.ReadAsStreamAsync();
+        using var streamReader = new StreamReader(responseStream);
+        string? line;
+        // until the stream ends ...
+        while((line = await streamReader.ReadLineAsync()) != null) {
+            if(string.IsNullOrWhiteSpace(line)) continue; // skip if line is empty
+
+            // llama returns streams in the format data: { "response" }, so ignore if otherwise
+            if(line.StartsWith("data: ")) {
+                var json = line.Substring(6); // clip the "data: " portion
+                try {
+                    // parse the json sent back
+                    using var doc = JsonDocument.Parse(json);
+                    var root = doc.RootElement;
+                    if(root.TryGetProperty("content", out var content)) {
+                        var text = content.GetString();
+                        if(!string.IsNullOrEmpty(text)) {
+                            // theres actually something here, so respond
+                            await Response.WriteAsync(text);
+                            await Response.Body.FlushAsync(); // rely outwards as quickly as it flows in
+                        }
+                    }
+                } catch {
+                    // malformed response data
+                }
+            }
+        }
+    }
+
+}
+
+public class PromptRequest {
+    public string Prompt { get; set; } = "";
+}

scripts/DEV_README.md

@@ -13,8 +13,8 @@ To see live logs:
 sudo docker logs -f -t agologum-api
 
 public user:
-> username=bard (admin)
+> username=bard (sensitive data modify permissions)
-> username=xvbard (superuser)
+> username=xvbard (sensitive data read permissions)
 > password=Public*890
 
 chrome dev tools troubleshooting
@@ -23,3 +23,4 @@ chrome dev tools troubleshooting
 Always test build before committing
 > for the client: $ npm run dev
 > for the api: $ dotnet build
+