diff --git a/api/Program.cs b/api/Program.cs index d4f8478..d4e6055 100644 --- a/api/Program.cs +++ b/api/Program.cs @@ -29,6 +29,8 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); // if this grows sufficiently large we can put elsewhere +// misc services that I didnt make +builder.Services.AddHttpClient(); // configuration for jwt authentication builder.Services.AddIdentity() @@ -110,7 +112,7 @@ if (app.Environment.IsEnvironment("Development")) { app.UseHttpsRedirection(); } -app.MapControllers(); +app.MapControllers(); // automatically scours src/Controllers for controller classes // attempt enitity-framework migrations at startup. love you stack overflow using (var scope = app.Services.CreateScope()) { @@ -125,7 +127,6 @@ using (var scope = app.Services.CreateScope()) { Thread.Sleep(5000); } } - } app.Run(); diff --git a/api/src/Controllers/ItemsController.cs b/api/src/Controllers/ItemsController.cs index 792a475..73b6851 100644 --- a/api/src/Controllers/ItemsController.cs +++ b/api/src/Controllers/ItemsController.cs @@ -6,9 +6,11 @@ using agologumApi.Models; using agologumApi.Services; [ApiController] -[Route("api/[controller]")] +[Route("api/[controller]")] // generally good practice to make this explicit public class ItemsController : ControllerBase { + // TODO: (global) controller's might eventually need more services so its gonna be a good idea to give them more meaningful names + // i.e. just name it itemService_ private readonly ItemService service_; public ItemsController(ItemService service) { diff --git a/api/src/Controllers/LlmController.cs b/api/src/Controllers/LlmController.cs new file mode 100644 index 0000000..566e023 --- /dev/null +++ b/api/src/Controllers/LlmController.cs @@ -0,0 +1,89 @@ + +// system usings +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Authorization; +using System.Net.Http; +using System.Text; +using System.Text.Json; +// agologum usings +using agologumApi.Models; +//using agologumApi.Services; + +[ApiController] +[Route("api/chat")] +public class LlmController : ControllerBase { + + // eventually this will be where most of the app sits + // might create src/Services/Llm/... to better organize + // private readonly LlmService llmService_; + private readonly IHttpClientFactory httpClientFactory_; + private readonly string LLAMA_URL = "http://localhost:8010/completion"; // TODO: can just put this in appsettings + + public LlmController(/* LlmService llmService, */ IHttpClientFactory httpClientFactory) { + // llmService_ = llmService; + httpClientFactory_ = httpClientFactory; + } + + // posting to api/chat/stream with a prompt in the body will return an sse (server sent events) stream of the llm text + // [Authorize(Policy = Permission.SensitiveData_Read)] // will make secret after testing + [HttpPost("stream")] + public async Task streamPrompt([FromBody] PromptRequest prompt) { + + var httpClient = httpClientFactory_.CreateClient(); // recreating this on every prompt request? idk seems 🐟y + + // json object for llama to parse + var llamaRequest = new { + prompt = prompt.Prompt, // alternative would be a messages array + stream = true, // real time token streaming + n_predict = 256 // max amount of tokens + // other options are temperature, top_p and top_k for vocabulary diversity, stop for stop tokens + // https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md + // llm loras lol + }; + + // http request to send to the llama server + var httpRequest = new HttpRequestMessage(HttpMethod.Post, LLAMA_URL) { + Content = new StringContent(JsonSerializer.Serialize(llamaRequest), Encoding.UTF8, "application/json") + }; + + // send request + var response = await httpClient.SendAsync(httpRequest, HttpCompletionOption.ResponseHeadersRead, HttpContext.RequestAborted); + response.EnsureSuccessStatusCode(); + Response.StatusCode = 200; + Response.ContentType = "text/plain"; // because its async dotnet lets us do this monstrosity + + // now to handle the response stream + await using var responseStream = await response.Content.ReadAsStreamAsync(); + using var streamReader = new StreamReader(responseStream); + string? line; + // until the stream ends ... + while((line = await streamReader.ReadLineAsync()) != null) { + if(string.IsNullOrWhiteSpace(line)) continue; // skip if line is empty + + // llama returns streams in the format data: { "response" }, so ignore if otherwise + if(line.StartsWith("data: ")) { + var json = line.Substring(6); // clip the "data: " portion + try { + // parse the json sent back + using var doc = JsonDocument.Parse(json); + var root = doc.RootElement; + if(root.TryGetProperty("content", out var content)) { + var text = content.GetString(); + if(!string.IsNullOrEmpty(text)) { + // theres actually something here, so respond + await Response.WriteAsync(text); + await Response.Body.FlushAsync(); // rely outwards as quickly as it flows in + } + } + } catch { + // malformed response data + } + } + } + } + +} + +public class PromptRequest { + public string Prompt { get; set; } = ""; +}