diff --git a/.acf/tasks.json b/.acf/tasks.json new file mode 100644 index 0000000..21e6fc8 --- /dev/null +++ b/.acf/tasks.json @@ -0,0 +1,446 @@ +{ + "projectName": "metal-kompanion-mcp", + "projectDescription": "MCP backend for Kompanion: memory/context/embedding provider over MCP, built from scratch (qtmcp-based) to persist conversation state and serve embeddings + retrieval to avoid forgetting across threads.", + "lastTaskId": 20, + "tasks": [ + { + "id": 1, + "title": "Project Setup: metal-kompanion-mcp", + "description": "MCP backend for Kompanion: memory/context/embedding provider over MCP, built from scratch (qtmcp-based) to persist conversation state and serve embeddings + retrieval to avoid forgetting across threads.", + "status": "todo", + "priority": 700, + "priorityDisplay": "high", + "dependsOn": [], + "createdAt": "2025-10-13T17:31:50.258Z", + "updatedAt": "2025-10-13T17:31:50.258Z", + "subtasks": [], + "relatedFiles": [], + "activityLog": [], + "lastSubtaskIndex": 0 + }, + { + "id": 2, + "title": "Design MCP memory/context API", + "description": "Specify MCP tools for: save_context, recall_context, embed_text, upsert_memory, search_memory, warm_cache. Define input/output schemas, auth, and versioning.", + "status": "in_progress", + "priority": 500, + "priorityDisplay": "P0", + "dependsOn": [], + "createdAt": "2025-10-13T17:32:24.705Z", + "updatedAt": "2025-10-13T17:40:02.144Z", + "subtasks": [ + { + "id": "2.1", + "title": "Write JSON Schemas for tools (done)", + "status": "todo", + "createdAt": "2025-10-13T17:39:21.256Z", + "updatedAt": "2025-10-13T17:39:21.256Z", + "activityLog": [ + { + "timestamp": "2025-10-13T17:39:21.256Z", + "type": "log", + "message": "Subtask created with title: \"Write JSON Schemas for tools (done)\"" + } + ] + } + ], + "lastSubtaskIndex": 1, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T17:32:24.705Z", + "type": "log", + "message": "Task created with title: \"Design MCP memory/context API\"" + }, + { + "timestamp": "2025-10-13T17:40:02.144Z", + "type": "log", + "message": "Status changed from \"todo\" to \"in_progress\". Message: Docs and schemas created. Proceeding to server scaffold and adapters." + } + ] + }, + { + "id": 3, + "title": "Select embedding backend & storage", + "description": "Choose between local (Ollama/gguf via llama.cpp embedding) vs remote (OpenAI/SentenceTransformers). Storage: sqlite+vectstore (pgvector/qdrant/chroma). Provide abstraction + adapters.", + "status": "todo", + "priority": 501, + "priorityDisplay": "P0", + "dependsOn": [], + "createdAt": "2025-10-13T17:32:35.110Z", + "updatedAt": "2025-10-13T17:32:35.110Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T17:32:35.110Z", + "type": "log", + "message": "Task created with title: \"Select embedding backend & storage\"" + } + ] + }, + { + "id": 4, + "title": "Scaffold qtmcp-based server", + "description": "Set up C++/Qt MCP server skeleton using qtmcp. Implement handshake, tool registration, and simple ping tool. Build with CMake in /home/kompanion/dev/metal/src/metal-kompanion.", + "status": "in_progress", + "priority": 499, + "priorityDisplay": "P1", + "dependsOn": [], + "createdAt": "2025-10-13T17:32:47.443Z", + "updatedAt": "2025-10-13T18:13:07.568Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T17:32:47.443Z", + "type": "log", + "message": "Task created with title: \"Scaffold qtmcp-based server\"" + }, + { + "timestamp": "2025-10-13T18:13:07.568Z", + "type": "log", + "message": "Status changed from \"todo\" to \"in_progress\". Message: Starting MCP server skeleton with tool registry, ping tool, and placeholders for kom.memory.v1 handlers." + } + ] + }, + { + "id": 5, + "title": "Implement memory adapters", + "description": "Adapters: (1) SQLite+FAISS/pgvector, (2) Qdrant, (3) Chroma. CRUD: upsert, delete, query, batch. Support namespaces (project/thread), TTL, metadata tags.", + "status": "todo", + "priority": 502, + "priorityDisplay": "P1", + "dependsOn": [], + "createdAt": "2025-10-13T17:32:57.756Z", + "updatedAt": "2025-10-13T17:32:57.756Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T17:32:57.756Z", + "type": "log", + "message": "Task created with title: \"Implement memory adapters\"" + } + ] + }, + { + "id": 6, + "title": "Deep research: memory DB architecture & schema", + "description": "Survey best practices for conversational memory stores (RAG, TTL, namespaces, versioning). Produce target schema for Postgres+pgvector and SQLite mappings.", + "status": "todo", + "priority": 498, + "priorityDisplay": "P0", + "dependsOn": [], + "createdAt": "2025-10-13T17:46:18.403Z", + "updatedAt": "2025-10-13T17:46:18.403Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T17:46:18.403Z", + "type": "log", + "message": "Task created with title: \"Deep research: memory DB architecture & schema\"" + } + ] + }, + { + "id": 7, + "title": "Decide primary DB: Postgres+pgvector vs SQLite+FAISS", + "description": "Evaluate tradeoffs (multi-user, concurrency, migrations, backups). Pick canonical prod DB and document local dev fallback.", + "status": "todo", + "priority": 503, + "priorityDisplay": "P0", + "dependsOn": [], + "createdAt": "2025-10-13T17:47:21.042Z", + "updatedAt": "2025-10-13T17:47:21.042Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T17:47:21.042Z", + "type": "log", + "message": "Task created with title: \"Decide primary DB: Postgres+pgvector vs SQLite+FAISS\"" + } + ] + }, + { + "id": 8, + "title": "Implement DAL + migrations (pgvector)", + "description": "Create C++ DAL layer for namespaces, items, chunks, embeddings. Add migration runner and seed scripts. Map MCP tool calls to DB ops.", + "status": "todo", + "priority": 497, + "priorityDisplay": "P1", + "dependsOn": [], + "createdAt": "2025-10-13T17:47:30.982Z", + "updatedAt": "2025-10-13T17:47:30.982Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T17:47:30.982Z", + "type": "log", + "message": "Task created with title: \"Implement DAL + migrations (pgvector)\"" + } + ] + }, + { + "id": 9, + "title": "Add cloud DB hardening (RLS, FTS/trgm, ANN indexes)", + "description": "Implement RLS policies; add FTS + pg_trgm for lexical search; unique (namespace_id, key); partial ANN indexes per model.", + "status": "todo", + "priority": 504, + "priorityDisplay": "P0", + "dependsOn": [], + "createdAt": "2025-10-13T19:13:13.769Z", + "updatedAt": "2025-10-13T19:13:13.769Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T19:13:13.769Z", + "type": "log", + "message": "Task created with title: \"Add cloud DB hardening (RLS, FTS/trgm, ANN indexes)\"" + } + ] + }, + { + "id": 10, + "title": "Server enforcement: scope injection + rate limits", + "description": "Inject namespace/user via session context; default-deny for scope widening; add simple per-tool rate limits.", + "status": "todo", + "priority": 496, + "priorityDisplay": "P0", + "dependsOn": [], + "createdAt": "2025-10-13T19:13:21.164Z", + "updatedAt": "2025-10-13T19:13:21.164Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T19:13:21.164Z", + "type": "log", + "message": "Task created with title: \"Server enforcement: scope injection + rate limits\"" + } + ] + }, + { + "id": 11, + "title": "Redaction & sensitivity pipeline", + "description": "Implement preprocessing to detect/seal secrets; set metadata.sensitivity; skip FTS/embeddings for `secret` items.", + "status": "todo", + "priority": 505, + "priorityDisplay": "P1", + "dependsOn": [], + "createdAt": "2025-10-13T19:13:29.391Z", + "updatedAt": "2025-10-13T19:13:29.392Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T19:13:29.391Z", + "type": "log", + "message": "Task created with title: \"Redaction & sensitivity pipeline\"" + } + ] + }, + { + "id": 12, + "title": "Private vault mode (key-only retrieval)", + "description": "Implement vault path for secret items: encrypted-at-rest only; disable participation in FTS/vector; key-based recall APIs.", + "status": "todo", + "priority": 495, + "priorityDisplay": "P1", + "dependsOn": [], + "createdAt": "2025-10-13T19:13:36.653Z", + "updatedAt": "2025-10-13T19:13:36.653Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T19:13:36.653Z", + "type": "log", + "message": "Task created with title: \"Private vault mode (key-only retrieval)\"" + } + ] + }, + { + "id": 13, + "title": "Local backup tools: export/import (E2EE)", + "description": "Add kom.local.v1.backup.export_encrypted / import_encrypted using the draft backup format.", + "status": "todo", + "priority": 506, + "priorityDisplay": "P1", + "dependsOn": [], + "createdAt": "2025-10-13T19:13:44.851Z", + "updatedAt": "2025-10-13T19:13:44.851Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T19:13:44.851Z", + "type": "log", + "message": "Task created with title: \"Local backup tools: export/import (E2EE)\"" + } + ] + }, + { + "id": 14, + "title": "Cloud adapters: backup/sync & payments stubs", + "description": "Expose kom.cloud.v1.backup.upload/restore, kom.cloud.v1.sync.push/pull, and payments.* stubs.", + "status": "todo", + "priority": 494, + "priorityDisplay": "P2", + "dependsOn": [], + "createdAt": "2025-10-13T19:13:55.490Z", + "updatedAt": "2025-10-13T19:13:55.490Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T19:13:55.490Z", + "type": "log", + "message": "Task created with title: \"Cloud adapters: backup/sync & payments stubs\"" + } + ] + }, + { + "id": 15, + "title": "Purge job & admin delete paths", + "description": "Implement scheduled hard-deletes for soft-deleted/expired items; add admin nuke namespace/user procedure.", + "status": "todo", + "priority": 507, + "priorityDisplay": "P2", + "dependsOn": [], + "createdAt": "2025-10-13T19:14:06.080Z", + "updatedAt": "2025-10-13T19:14:06.080Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T19:14:06.080Z", + "type": "log", + "message": "Task created with title: \"Purge job & admin delete paths\"" + } + ] + }, + { + "id": 16, + "title": "Test suite: privacy & hybrid search", + "description": "Cross-tenant leakage, redaction invariants, TTL/purge, lexical vs hybrid parity, hosted vs local parity.", + "status": "todo", + "priority": 493, + "priorityDisplay": "P0", + "dependsOn": [], + "createdAt": "2025-10-13T19:14:14.309Z", + "updatedAt": "2025-10-13T19:14:14.310Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T19:14:14.310Z", + "type": "log", + "message": "Task created with title: \"Test suite: privacy & hybrid search\"" + } + ] + }, + { + "id": 17, + "title": "Enable Qwen-2.5-Coder with tool support (Happy-Code profile)", + "description": "Prepare system prompt + registry injection + JSON-only protocol enforcement; provide tool schemas and example transcripts; validate with kom.memory/local backup tools.", + "status": "todo", + "priority": 508, + "priorityDisplay": "P0", + "dependsOn": [], + "createdAt": "2025-10-13T23:29:36.547Z", + "updatedAt": "2025-10-13T23:29:36.548Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T23:29:36.548Z", + "type": "log", + "message": "Task created with title: \"Enable Qwen-2.5-Coder with tool support (Happy-Code profile)\"" + } + ] + }, + { + "id": 18, + "title": "Expose Agentic-Control-Framework as a tool", + "description": "Wrap ACF endpoints into a tool registry accessible to models (list/add/update tasks, read/write files, run commands) with strict allowlist per workspace.", + "status": "todo", + "priority": 492, + "priorityDisplay": "P0", + "dependsOn": [], + "createdAt": "2025-10-13T23:29:43.303Z", + "updatedAt": "2025-10-13T23:29:43.304Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T23:29:43.304Z", + "type": "log", + "message": "Task created with title: \"Expose Agentic-Control-Framework as a tool\"" + } + ] + }, + { + "id": 19, + "title": "DAL skeleton + SQL calls (pgvector)", + "description": "Create DAL interfaces and pgvector implementation stubs: connect, begin/commit, upsert item/chunk/embedding, search (text+vector placeholder), prepared SQL in sql/pg. Wire handlers to DAL in no-op mode.", + "status": "todo", + "priority": 509, + "priorityDisplay": "P0", + "dependsOn": [], + "createdAt": "2025-10-13T23:29:49.918Z", + "updatedAt": "2025-10-13T23:29:49.918Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-13T23:29:49.918Z", + "type": "log", + "message": "Task created with title: \"DAL skeleton + SQL calls (pgvector)\"" + } + ] + }, + { + "id": 20, + "title": "Claude Code integration rescue plan", + "description": "Stabilize Qwen-2.5-Coder inside Claude Code despite heavy system prompts: hard system override, JSON-only protocol, stop-sequences, tool registry injection, and fallback DSL.", + "status": "todo", + "priority": 491, + "priorityDisplay": "P0", + "dependsOn": [], + "createdAt": "2025-10-14T00:06:04.896Z", + "updatedAt": "2025-10-14T00:06:04.896Z", + "subtasks": [], + "lastSubtaskIndex": 0, + "relatedFiles": [], + "activityLog": [ + { + "timestamp": "2025-10-14T00:06:04.896Z", + "type": "log", + "message": "Task created with title: \"Claude Code integration rescue plan\"" + } + ] + } + ] +} \ No newline at end of file diff --git a/.cursor/rules/task_manager_workflow.mdc b/.cursor/rules/task_manager_workflow.mdc new file mode 100644 index 0000000..a2f34c2 --- /dev/null +++ b/.cursor/rules/task_manager_workflow.mdc @@ -0,0 +1,7 @@ +# Cursor AI Workflow Rules for Agentic Control Framework + +# (Define rules here to tell Cursor how to use acf commands) + +# Example: +# To list tasks, use the command: acf list +# To get the next task: acf next diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..13c680d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,35 @@ +# Repository Guidelines +This guide supports new agents contributing to `metal-kompanion`, the MCP backend for Kompanion. Follow these practices to keep the service buildable, testable, and easy to review. + +## Project Structure & Module Organization +- `src/` holds C++ code: `mcp/` for server facade and tool routing, `memory/` for embeddings contracts, `dal/` for persistence, and `policy/` for capability rules. +- `docs/` hosts design notes; `runtime/kom_runner.py` is the Python orchestrator for agent execution against Ollama; `db/` and `sql/` capture Postgres schemas. +- Place sample payloads or fixtures under `tests/` (currently `tests/schemas/` for JSON samples). Generated artifacts live in `build/`. + +## Build, Test, and Development Commands +```bash +cmake -S . -B build # configure with CMake 3.22+, targets C++20 +cmake --build build -j # compile the kom_mcp executable +ctest --test-dir build # run CTest suites once they are defined +python3 runtime/kom_runner.py # exercise the runtime loop (requires OLLAMA_BASE) +``` +Run the Python runtime from a virtualenv seeded with `pip install -r runtime/requirements.txt`. + +## Coding Style & Naming Conventions +- C++20, clang/gcc friendly, 4-space indentation, braces on the same line (`class KomMcpServer {`). +- Classes use PascalCase; methods camelCase; private members keep the trailing underscore (`tools_`). Prefer `std::` types and RAII helpers over raw pointers. +- Keep headers lean: forward declare where possible and document non-obvious behavior with concise comments. + +## Testing Guidelines +- Add unit or schema validation tests under `tests/`, mirroring the source tree (e.g., `tests/mcp/` for dispatcher tests). +- Register new tests with CTest via `add_test` in `CMakeLists.txt`; verify they pass with `ctest --output-on-failure`. +- Provide realistic JSON samples for new tools alongside schema updates to guard against regressions. + +## Commit & Pull Request Guidelines +- Follow the existing `scope: message` pattern from git history (`docker: fix host ollama port`); keep messages imperative and present tense. +- Each PR should state intent, link relevant issues, and include before/after notes or screenshots when UI-adjacent runtime behavior changes. +- Mention how to reproduce test runs (`cmake --build`, `ctest`) and flag configuration or migration steps (e.g., `sql/` changes). + +## Security & Configuration Tips +- Do not commit secrets; runtime state lives beneath XDG paths (`~/.local/state/kompanion`). Document any new env vars in `docs/`. +- When integrating new tools, ensure access control aligns with `policy/` capabilities and record expected journal entries in `runtime` docs. diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f4927c..9514f04 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,10 +1,20 @@ -cmake_minimum_required(VERSION 3.16) -project(MetalKompanion LANGUAGES CXX) +cmake_minimum_required(VERSION 3.22) +project(metal_kompanion_mcp LANGUAGES CXX) + set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) -find_package(Qt6 REQUIRED COMPONENTS Core Network DBus) -add_executable(kompanion_server - src/main.cpp -) -target_link_libraries(kompanion_server Qt6::Core Qt6::Network Qt6::DBus) -install(TARGETS kompanion_server RUNTIME DESTINATION bin) + +# Placeholder: find Qt and qtmcp when available +# find_package(Qt6 COMPONENTS Core Network REQUIRED) +# find_package(qtmcp REQUIRED) + +add_executable(kom_mcp src/main.cpp) +# target_link_libraries(kom_mcp PRIVATE Qt6::Core Qt6::Network qtmcp) + +install(TARGETS kom_mcp RUNTIME DESTINATION bin) + +add_executable(test_mcp_tools tests/contract/test_mcp_tools.cpp) +target_include_directories(test_mcp_tools PRIVATE src) + +enable_testing() +add_test(NAME contract_mcp_tools COMMAND test_mcp_tools) diff --git a/README.md b/README.md new file mode 100644 index 0000000..0107a4e --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +# metal-kompanion-mcp + +MCP backend and memory provider for Kompanion. Uses `qtmcp` (Qt-based MCP) to expose tools under namespace `kom.memory.v1`. + +## Build +```bash +cmake -S . -B build +cmake --build build -j +``` + +## Layout +- `src/main.cpp` – entry point (stub until qtmcp wired) +- `src/mcp/ToolSchemas.json` – JSON Schemas for MCP tools +- `src/memory/` – interfaces for embedder and vector store +- `docs/` – design notes + +## Next +- Add qtmcp dependency and implement server with tool registration. +- Implement adapters: embedder(s) + vector store(s). +- Wire `save_context`/`recall_context` to persistent store. diff --git a/bin/compose-up-host.sh b/bin/compose-up-host.sh new file mode 100755 index 0000000..a4303be --- /dev/null +++ b/bin/compose-up-host.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +docker compose -f docker/compose.host.yml up -d diff --git a/db/queries.sql b/db/queries.sql new file mode 100644 index 0000000..083e46a --- /dev/null +++ b/db/queries.sql @@ -0,0 +1 @@ +-- Kompanion quick checks diff --git a/db/schema_branch_embeddings.sql b/db/schema_branch_embeddings.sql new file mode 100644 index 0000000..d4df567 --- /dev/null +++ b/db/schema_branch_embeddings.sql @@ -0,0 +1 @@ +-- placeholder: see chat for full Branch Embeddings schema diff --git a/db/schema_dev.sql b/db/schema_dev.sql new file mode 100644 index 0000000..e69de29 diff --git a/db/schema_pgvector.sql b/db/schema_pgvector.sql new file mode 100644 index 0000000..5b1be28 --- /dev/null +++ b/db/schema_pgvector.sql @@ -0,0 +1,5 @@ +-- pgvector knowledge schema (v0.2, branch embeddings)nCREATE EXTENSION IF NOT EXISTS vector;nn-- Documents tracked by content hash; multiple logical paths may point to same blobnCREATE TABLE IF NOT EXISTS doc_blob (n blob_sha256 text PRIMARY KEY,n bytes bigint NOT NULL,n created_at timestamptz DEFAULT now()n);nn-- Logical document identity and branch/commit lineagenCREATE TABLE IF NOT EXISTS doc (n id bigserial PRIMARY KEY,n origin text, -- e.g., filesystem, forgejo, exportn root_path text, -- repo root or base dirn rel_path text, -- path relative to rootn blob_sha256 text REFERENCES doc_blob(blob_sha256)n);nCREATE TABLE IF NOT EXISTS branch (n id bigserial PRIMARY KEY,n name text UNIQUE,n created_at timestamptz DEFAULT now()n);nCREATE TABLE IF NOT EXISTS revision (n id bigserial PRIMARY KEY,n branch_id bigint REFERENCES branch(id) ON DELETE CASCADE,n parent_id bigint REFERENCES revision(id),n commit_sha text, -- optional link to VCSn message text,n created_at timestamptz DEFAULT now()n);nn-- Chunked text with provenance; each revision can re-chunk same docnCREATE TABLE IF NOT EXISTS chunk (n id bigserial PRIMARY KEY,n revision_id bigint REFERENCES revision(id) ON DELETE CASCADE,n doc_id bigint REFERENCES doc(id) ON DELETE CASCADE,n lineno int,n text text NOT NULL,n tags text[],n sha256 text,n created_at timestamptz DEFAULT now()n);nCREATE INDEX IF NOT EXISTS idx_chunk_doc ON chunk(doc_id);nCREATE INDEX IF NOT EXISTS idx_chunk_rev ON chunk(revision_id);nn-- Embedding models registrynCREATE TABLE IF NOT EXISTS embed_model (n id bigserial PRIMARY KEY,n name text UNIQUE NOT NULL, -- e.g., mxbai-embed-largen dims int NOT NULL,n tokenizer text,n notes textn);nn-- Branch embeddings: per revision, per chunk, per model (dims parametrized by table)nCREATE TABLE IF NOT EXISTS embedding_1024 (n id bigserial PRIMARY KEY,n chunk_id bigint REFERENCES chunk(id) ON DELETE CASCADE,n model_id bigint REFERENCES embed_model(id),n embedding vector(1024) NOT NULL,n created_at timestamptz DEFAULT now()n);nCREATE INDEX IF NOT EXISTS idx_emb1024_chunk ON embedding_1024(chunk_id);nCREATE INDEX IF NOT EXISTS idx_emb1024_vec ON embedding_1024 USING ivfflat (embedding vector_l2_ops) WITH (lists=100);nn-- Views: latest chunks per branchnCREATE OR REPLACE VIEW v_latest_chunks AS +SELECT c.* FROM chunk c +JOIN (SELECT revision_id, max(id) over (partition by doc_id, branch_id) as maxid, doc_id + FROM revision r JOIN chunk c2 ON c2.revision_id=r.id) t + ON c.id=t.maxid;nn-- Ledger mirror (optional)nCREATE TABLE IF NOT EXISTS ledger_mirror (n id bigserial PRIMARY KEY,n ts timestamptz NOT NULL,n actor text,n action text,n body jsonbn);n \ No newline at end of file diff --git a/db/schema_runtime.sql b/db/schema_runtime.sql new file mode 100644 index 0000000..e69de29 diff --git a/docs/IDENTITY_AND_ASPECTS.md b/docs/IDENTITY_AND_ASPECTS.md new file mode 100644 index 0000000..14e9427 --- /dev/null +++ b/docs/IDENTITY_AND_ASPECTS.md @@ -0,0 +1 @@ +# Identity & Aspects (placeholder) diff --git a/docs/RUNTIME_VS_PATTERN_EXCHANGE.md b/docs/RUNTIME_VS_PATTERN_EXCHANGE.md new file mode 100644 index 0000000..bad4481 --- /dev/null +++ b/docs/RUNTIME_VS_PATTERN_EXCHANGE.md @@ -0,0 +1 @@ +# Runtime vs Pattern Exchange (placeholder) diff --git a/docs/backup-e2ee-spec.md b/docs/backup-e2ee-spec.md new file mode 100644 index 0000000..7d684f8 --- /dev/null +++ b/docs/backup-e2ee-spec.md @@ -0,0 +1,25 @@ +# Encrypted Backup Format (Draft) + +> *Cipher left open by design (to be decided: AES-GCM vs XChaCha20-Poly1305; key storage via OS keychain or passphrase vault).* + +## Goals +- Client-side encrypted backups (no cleartext leaves device). +- Content-addressed chunks with manifest; resumable upload. +- Key rotation support and device enrollment flow. + +## Artifacts +- `manifest.json` — version, namespace(s), chunks, sizes, hashes, KDF params, encryption metadata. +- `payload.tar.zst` — concatenated content/chunks (encrypted). + +## Flow +1. Collect items/chunks for selected namespaces. +2. Serialize → compress → encrypt → write manifest. +3. Upload manifest + blob via provider adapter (e.g., Google Drive) as opaque objects. +4. Restore: download → decrypt → verify hashes → import. + +## Provider Adapters +- `kom.local.v1.backup.export_encrypted` / `import_encrypted` (local). +- `kom.cloud.v1.backup.upload` / `restore` (remote; encrypted blobs only). + +## Open Questions +- Final cipher/KDF choices, key wrapping and rotation UX, multi-namespace packaging. diff --git a/docs/claude-code-hard-overrides.md b/docs/claude-code-hard-overrides.md new file mode 100644 index 0000000..80dbdc2 --- /dev/null +++ b/docs/claude-code-hard-overrides.md @@ -0,0 +1,32 @@ +# Claude Code: Hard Overrides to Stabilize Tool Use + +When Claude Code's layered system prompts clash with our tool protocol, force a minimal, deterministic lane. + +## Settings (recommended) +- **Stop sequences**: ` +````, ` + +` (or whatever the IDE uses to inject formatting). +- **Max output tokens**: small (e.g., 512). +- **Temperature**: 0.1-0.3. +- **Disable auto-formatting / code fences** if possible. +- **Disable auto-tool use**; we trigger tools via explicit JSON only. + +## System message (short) +Use the contents of `docs/prompts/qwen_tool_mode_system.txt` verbatim as the *final* system layer closest to the model. + +## Runtime guardrails +- Reject any non-JSON output; send a short corrective user message: `OUTPUT MUST BE JSON. Please resend.` +- If repeated, send `{"final":{"content":{"error":"RESET_REQUIRED"}}}` back and restart the session. + +## Registry injection +- Provide the tool list and JSON Schemas (kom.memory.*, kom.local.backup.*, acf.*). +- Keep it short; link to full schemas if the UI allows references. + +## Troubleshooting +- If model keeps adding prose, tighten stop sequences and lower max tokens. +- If JSON keys drift, include a 2–3 line example of a **valid** `action` and a **valid** `final`. +- If it calls undefined tools, respond with a single tool error and re-present the allowlist. + +## Fallback DSL +- Accept `@tool {json-args}` and convert to a JSON `action` behind the scenes when necessary. diff --git a/docs/dal-skeleton.md b/docs/dal-skeleton.md new file mode 100644 index 0000000..9e5baf5 --- /dev/null +++ b/docs/dal-skeleton.md @@ -0,0 +1,19 @@ +# DAL Skeleton (pgvector) + +## Interfaces +- `IDatabase` — connect/tx + memory ops (ensureNamespace, upsertItem/Chunks/Embeddings, searchText/searchVector). +- `PgDal` — stub implementation for now (no libpq linked). + +## SQL Calls (target) +- ensureNamespace: `INSERT ... ON CONFLICT (name) DO UPDATE RETURNING id` +- upsertItem: `INSERT ... ON CONFLICT (id) DO UPDATE SET ... RETURNING id` +- upsertChunks: batch insert w/ `RETURNING id` +- upsertEmbeddings: batch insert; ensure `model, dim` set, vector column populated. +- searchText: FTS/trigram query filtered by namespace/thread/tags. +- searchVector: `ORDER BY embeddings.vector <-> $1 LIMIT k` (with filters). + +## Next +- Wire `Handlers::upsert_memory` / `search_memory` to `IDatabase`. +- Add libpq (or pqxx) and parameterized statements. +- Add RLS/session GUCs & retries. + diff --git a/docs/db-schema.md b/docs/db-schema.md new file mode 100644 index 0000000..72b8b07 --- /dev/null +++ b/docs/db-schema.md @@ -0,0 +1,36 @@ +# Target Database Schema for Kompanion Memory (v0) + +**Primary**: Postgres 14+ with `pgvector` (v0.6+) for embeddings. +**Alt**: SQLite 3 + FAISS (local dev / fallback). + +## Design Principles +- **Namespaces** (`project:user:thread`) partition memory and enable scoped retrieval. +- **Separation of items vs chunks**: items are logical notes/contexts; chunks are embedding units. +- **Metadata-first**: JSONB metadata with selective indexed keys; tags array. +- **Retention**: TTL via `expires_at`; soft-delete via `deleted_at`. +- **Versioning**: monotonically increasing `revision`; latest view via upsert. +- **Observability**: created/updated audit, model/dim for embeddings. + +## Entities +- `namespaces` – registry of logical scopes. +- `threads` – optional conversational threads within a namespace. +- `users` – optional association to user identity. +- `memory_items` – logical items with rich metadata and raw content. +- `memory_chunks` – embedding-bearing chunks derived from items. +- `embeddings` – embedding vectors (one per chunk + model info). + +## Retrieval Flow +1) Query text → embed → ANN search on `embeddings.vector` (filtered by namespace/thread/tags/metadata). +2) Join back to `memory_items` to assemble content and metadata. + +## Indexing +- `embeddings`: `USING ivfflat (vector) WITH (lists=100)` (tune), plus btree on `(model, dim)`. +- `memory_items`: GIN on `metadata`, GIN on `tags`, btree on `(namespace_id, thread_id, created_at)`. + +## SQLite Mapping +- Same tables sans vector column; store vectors in a sidecar FAISS index keyed by `chunk_id`. Maintain consistency via triggers in app layer. + +## Open Questions +- Hybrid search strategy (BM25 + vector) — defer to v1. +- Eventing for cache warms and eviction. +- Encryption at rest and PII handling. diff --git a/docs/design-local-first-architecture.md b/docs/design-local-first-architecture.md new file mode 100644 index 0000000..536ba3f --- /dev/null +++ b/docs/design-local-first-architecture.md @@ -0,0 +1,56 @@ +# Design Decision: Local-First Personal Store with Optional Federated Services + +**Decision**: Kompanion adopts a **two-tier architecture**. A personal, local store (Akonadi-like) is the *authoritative home* of a user's data and operates fully offline. An optional federated layer provides encrypted backups, multi-device sync, and paid cloud conveniences (e.g., hosted search/rerank). Users can run **purely local**, or selectively enable cloud features. + +**Encryption Note**: We deliberately leave the *exact cryptography suite* open to allow hardware/OS keychains, libsodium, AES-GCM, or XChaCha20-Poly1305. The guardrails below assume **end-to-end encryption (E2EE)** with keys controlled by the user. + +--- + +## 1) Personal Store (Local Core) — `kom.local.v1` +- Runs entirely on-device; no network required. +- DB: SQLite (+ FTS/trigram for "rgrep" feel) + FAISS for vectors. +- Embeddings/Reranker: local (Ollama + optional local reranker). +- Privacy defaults: do-not-embed secrets; private-vault items are never vectorized/FTS'd; E2EE for backups/exports. +- Backup tools: `backup.export_encrypted`, `backup.import_encrypted` (E2EE blobs). + +## 2) Federated Services (Optional) — `kom.cloud.v1` +- Adds encrypted sync, cloud backup, micropayment-backed hosted compute (e.g., heavy reranking), and optional hosted pgvector search. +- Server sees ciphertext plus minimal metadata; hosted search is opt-in and may store embeddings either encrypted or plaintext **only by explicit consent**. +- Per-namespace tenancy and isolation (RLS when using Postgres). + +## 3) Key & Auth Model +- Users may **only retain authentication/secret-store access**; Kompanion handles day-to-day operations. +- Device enrollment shares/wraps keys securely (mechanism TBD; QR/device handoff). +- Key rotation and export are first-class; backups are always encrypted client-side. + +## 4) Search Modes +- **Lexical**: FTS + trigram, scoped to namespace/thread/user; grep-like snippets. +- **Semantic**: vector ANN with local reranker by default. +- **Hybrid**: configurable orchestration; always respects scope and privacy flags. + +## 5) Privacy Controls +- Sensitivity flags: `metadata.sensitivity = secret|private|normal`. +- `secret` items: E2EE only (no FTS, no embeddings). +- Server-side scope injection (namespace/user) in all handlers; default-deny posture. +- Purge policy: soft-delete + scheduled hard-delete; cascades to chunks/embeddings and remote copies. + +## 6) Compatibility with Postgres+pgvector +- When cloud search is enabled, a hosted Postgres+pgvector instance enforces isolation via RLS and per-namespace session GUCs. +- Local SQLite store remains the source of truth unless user opts to delegate search to cloud. + +--- + +## Action List (from privacy review) +1. **DB hardening (cloud path)**: add RLS policies; add FTS + pg_trgm; unique `(namespace_id, key)`; partial ANN indexes per model. +2. **Server enforcement**: inject namespace/user via session context (GUCs); default-deny widening; rate limits. +3. **Redaction pipeline**: protect secrets before embedding; skip embedding/FTS for `secret` items. +4. **Private vault mode**: key-only retrieval paths for sensitive items (no index participation). +5. **Backups**: define E2EE export/import format; provider adapters (e.g., Google Drive) use pre-encrypted blobs. +6. **Sync**: event-log format (append-only); conflict rules; device enrollment + key wrapping; later CRDT if needed. +7. **Purging**: scheduled hard-deletes; admin "nuke namespace/user" procedure. +8. **Tests**: cross-tenant leakage, redaction invariants, purge/TTL, hybrid-vs-lexical, hosted-vs-local parity. + +## Files to Watch +- `docs/db-schema.md`, `sql/pg/001_init.sql` (cloud path) +- `src/mcp/ToolSchemas.json` and MCP handlers (scope + sensitivity gates) +- `kom.local.v1.backup.*`, `kom.cloud.v1.*` (new tool surfaces) diff --git a/docs/embeddings-and-storage.md b/docs/embeddings-and-storage.md new file mode 100644 index 0000000..9232d28 --- /dev/null +++ b/docs/embeddings-and-storage.md @@ -0,0 +1,26 @@ +# Embedding Backends & Storage Options + +## Embeddings +- **Local**: Ollama + llama.cpp gguf (e.g., nomic-embed-text, all-MiniLM-gguf). +- **Remote**: OpenAI text-embedding-3-small/large; SentenceTransformers via Python service. + +### Abstraction +`IEmbedder` interface with `embed(texts: string[], model?: string) -> {model, vectors}`. + +## Storage +- **SQLite + FAISS** (local, simple) +- **Postgres + pgvector** (robust, SQL) +- **Qdrant** (fast ANN, tags) +- **Chroma** (lightweight) + +### Abstraction +`IVectorStore` with `upsert`, `query`, `delete`, supports `namespace`, `metadata` filter and TTL. + +## Selection Matrix +- If offline-first: SQLite+FAISS + local embedder. +- If multi-host: Postgres+pgvector or Qdrant. + +## Next Steps +- Implement `IEmbedder` adapters (Ollama, OpenAI). +- Implement `IVectorStore` adapters (SQLite+FAISS, pgvector, Qdrant). +- Wire to MCP tools in `kom.memory.v1`. diff --git a/docs/mcp-memory-api.md b/docs/mcp-memory-api.md new file mode 100644 index 0000000..bf5a598 --- /dev/null +++ b/docs/mcp-memory-api.md @@ -0,0 +1,51 @@ +# MCP Memory/Context API for Kompanion + +## Goals +- Persist long-term context across threads/sessions. +- Provide embeddings + retrieval over namespaces (project/user/thread). +- Expose via MCP tools with JSON schemas; versioned and testable. + +## Tools +### `save_context` +Persist a context blob with metadata. +- input: `{ namespace: string, key?: string, content: any, tags?: string[], ttl_seconds?: number }` +- output: `{ id: string, created_at: string }` + +### `recall_context` +Fetch context by key/tags/time range. +- input: `{ namespace: string, key?: string, tags?: string[], limit?: number, since?: string }` +- output: `{ items: Array<{id:string, key?:string, content:any, tags?:string[], created_at:string}> }` + +### `embed_text` +Return vector embedding for given text(s). +- input: `{ model?: string, texts: string[] }` +- output: `{ model: string, vectors: number[][] }` + +### `upsert_memory` +Upsert text+metadata into vector store. +- input: `{ namespace: string, items: Array<{id?:string, text:string, metadata?:object, embedding?:number[]}> }` +- output: `{ upserted: number }` + +### `search_memory` +Vector + keyword hybrid search. +- input: `{ namespace: string, query: { text?: string, embedding?: number[], k?: number, filter?: object } }` +- output: `{ matches: Array<{id:string, score:number, text?:string, metadata?:object}> }` + +### `warm_cache` +Precompute embeddings for recent items. +- input: `{ namespace: string, since?: string }` +- output: `{ queued: number }` + +## Auth & Versioning +- toolNamespace: `kom.memory.v1` +- auth: bearer token via MCP session metadata (optional local mode). + +## Error Model +`{ error: { code: string, message: string, details?: any } }` + +## Events (optional) +- `memory.updated` broadcast over MCP notifications. + +## Notes +- Namespaces: `project:metal`, `thread:`, `user:`. +- Store raw content and normalized text fields for RAG. diff --git a/docs/memory-architecture.md b/docs/memory-architecture.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/plan-qwen-tools.md b/docs/plan-qwen-tools.md new file mode 100644 index 0000000..e9ae640 --- /dev/null +++ b/docs/plan-qwen-tools.md @@ -0,0 +1,27 @@ +# Plan: Qwen-2.5-Coder with Tool Support + ACF Exposure + +## Goals +- Make Qwen-2.5-Coder reliably call tools in Happy-Code. +- Expose Agentic-Control-Framework (ACF) as a safe tool registry to the model. +- Keep a fallback protocol (JSON-only) for models lacking native tools. + +## Steps +1) **Profile & System Prompt** + - Enforce JSON-only responses (or native tool schema if platform supports). + - Inject tool registry and JSON Schemas (kom.memory/local backup + ACF subset). +2) **Registry** + - Allowlist: `kom.memory.v1.*`, `kom.local.v1.backup.*`, and `acf.*` wrapper. + - Reject unknown tools and args mismatches (runtime guard). +3) **ACF as Tools** + - Map ACF endpoints: `acf.list_tasks`, `acf.add_task`, `acf.update_task`, `acf.read_file`, `acf.write_file`, `acf.exec`. + - Require workspace path + pattern allowlists. +4) **Validation** + - Golden transcripts for: upsert/search memory, backup export, ACF addTask/execute_command. +5) **Observability** + - Log tool calls (names + durations, no payloads). + +## Deliverables +- `docs/tool-calling-without-native-support.md` (done) +- `docs/plan-qwen-tools.md` (this) +- Happy-Code profile snippet for Qwen-2.5-Coder +- ACF tool wrapper module (C++ or Python) diff --git a/docs/prompts/qwen_tool_mode_system.txt b/docs/prompts/qwen_tool_mode_system.txt new file mode 100644 index 0000000..1ff95cb --- /dev/null +++ b/docs/prompts/qwen_tool_mode_system.txt @@ -0,0 +1,19 @@ +You are Qwen-2.5-Coder operating in TOOL MODE. + +CONTRACT: +- Always respond with a SINGLE JSON object. No prose, no markdown. +- Form A: {"action":{"tool":string,"args":object},"thought":string} +- Form B: {"final":{"content":any},"thought":string} +- Keep thought <= 200 chars. +- Only call tools from the provided REGISTRY & SCHEMAS. +- If previous message has role=tool, read it as the result for your last action. +- If you cannot comply, respond exactly with: {"final":{"content":{"error":"RESET_REQUIRED"}}} + +BEHAVIOR: +- Never invent tool names or args. +- Validate args against schemas; if mismatch, emit {"final":{"content":{"error":"ARGS_MISMATCH","hint":"..."}}}. +- Prefer minimal steps: upsert/search → final. +- Do not echo schemas or large content. + +OUTPUT ONLY JSON. + diff --git a/docs/sync-manifest.md b/docs/sync-manifest.md new file mode 100644 index 0000000..57ce529 --- /dev/null +++ b/docs/sync-manifest.md @@ -0,0 +1,30 @@ +# E2EE Sync Manifest (Draft) + +## Goals +- Multi-device E2EE sync with append-only event log. +- Minimal metadata on server (sizes, hashes, timestamps). + +## Event Types +- `item.upsert` (id, namespace_id, revision, metadata, content_ref?) +- `item.delete` (id) +- `chunk.add` (chunk_id, item_id, ord, text_ref?) +- `chunk.remove` (chunk_id) +- `embedding.add` (chunk_id, model, dim, vector_ref?) + +> _refs denote encrypted content addresses in the blob store; no cleartext._ + +## Conflict Rules +- Items: last-writer-wins per field; later CRDT as needed. +- Deleted beats update after a window. + +## Keys +- Device enrollment shares wrapped keys (mechanism TBD). +- Rotation supported via manifest updates and re-wrap. + +## MCP Surfaces +- `kom.cloud.v1.sync.push` / `pull` +- `kom.cloud.v1.backup.upload` / `restore` +- `kom.local.v1.backup.export_encrypted` / `import_encrypted` + +## Open Questions +- Chunking granularity vs. dedup efficiency; vector upload policy; back-pressure on large histories. diff --git a/docs/tool-calling-without-native-support.md b/docs/tool-calling-without-native-support.md new file mode 100644 index 0000000..1272356 --- /dev/null +++ b/docs/tool-calling-without-native-support.md @@ -0,0 +1,75 @@ +# Tool Calling Without Native Support (Happy-Code Compatibility) + +When models lack built-in tool/function calling, we can still get reliable tool use via a **protocol-in-prompt** approach plus a thin runtime. + +## 1) JSON Tool Protocol (deterministic) +Model must respond with a single JSON object, no prose. + +```json +{ + "thought": "", + "action": { + "tool": "", + "args": { } + } +} +``` + +- If more steps are needed, the runtime feeds the tool's JSON result back as the next user message with role=tool. +- The model must then either emit another action or finish with: + +```json +{ + "final": { "content": { } } +} +``` + +### Guardrails +- Reject any output that isn't valid JSON or that contains extra text. +- Cap thought to ~200 chars. +- Disallow calling tools not in the registry. + +## 2) Command DSL (fallback) +If JSON parsing is brittle, accept a single line command and parse it: + +@tool {json-args} + +Example: @tool kom.memory.v1.search_memory {"namespace":"project:metal", "query":{"text":"embedding model"}} + +## 3) Prompt Template (drop-in) +Use this system message for happy-code/claude-code sessions: + +You are a coding assistant that can call tools via a JSON protocol. Available tools (names & schema will be provided). Always reply with a single JSON object. No markdown. No commentary. Use this schema: { "thought": string (short), "action": { "tool": string, "args": object } } OR { "final": { "content": any } }. If a tool result is needed, emit action. If done, emit final. Never invent tool names or fields. + +## 4) Minimal Runtime (pseudocode) +while True: + msg = llm(messages) + data = json.loads(msg) + if 'action' in data: + tool = registry[data['action']['tool']] + result = tool(**data['action']['args']) + messages.append({"role":"tool","name":tool.name,"content":json.dumps(result)}) + continue + elif 'final' in data: + return data['final']['content'] + else: + error("Invalid protocol") + +## 5) MCP Integration +- Map registry to MCP tools (e.g., kom.memory.v1.*). +- Provide each tool's JSON Schema to the model in the system message (strict). + +## 6) Testing Checklist +- Invalid JSON → reject → ask to resend. +- Unknown tool → reject. +- Args mismatch → show schema snippet, ask to correct. +- Multi-step flows → verify tool result is consumed in next turn. + +## 7) Example Session +System: (template above + list of tools & schemas) +User: Save this note: "Embedding model comparison takeaways" into project:metal +Assistant: +{"thought":"need to upsert note","action":{"tool":"kom.memory.v1.upsert_memory","args":{"namespace":"project:metal","items":[{"text":"Embedding model comparison takeaways"}]}}} +Tool (kom.memory.v1.upsert_memory): { "upserted": 1 } +Assistant: +{"final":{"content":{"status":"ok","upserted":1}}} diff --git a/ledger.md b/ledger.md new file mode 100644 index 0000000..912cf0a --- /dev/null +++ b/ledger.md @@ -0,0 +1,5 @@ +# Ledger + +- 2025-10-13: Initialized project `metal-kompanion-mcp`; created docs and interfaces; scaffolded CMake and main stub. +- 2025-10-13: Added MCP tool schemas for `kom.memory.v1`. +- 2025-10-13: Built MCP skeleton with `ping` and `embed_text` stub; added local-first architecture docs; added backup/sync draft specs; created tasks for privacy hardening and cloud adapters. diff --git a/sql/pg/001_init.sql b/sql/pg/001_init.sql new file mode 100644 index 0000000..a14c7d9 --- /dev/null +++ b/sql/pg/001_init.sql @@ -0,0 +1,86 @@ +-- Enable pgvector (requires extension installed) +CREATE EXTENSION IF NOT EXISTS vector; + +-- Namespaces: unique logical scope (e.g., 'project:metal', 'thread:abc') +CREATE TABLE IF NOT EXISTS namespaces ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Users (optional link) +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + external_id TEXT UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Threads (within a namespace) +CREATE TABLE IF NOT EXISTS threads ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + namespace_id UUID NOT NULL REFERENCES namespaces(id) ON DELETE CASCADE, + external_id TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS threads_ns_idx ON threads(namespace_id); + +-- Memory items: logical notes/contexts (JSONB content + normalized text) +CREATE TABLE IF NOT EXISTS memory_items ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + namespace_id UUID NOT NULL REFERENCES namespaces(id) ON DELETE CASCADE, + thread_id UUID REFERENCES threads(id) ON DELETE SET NULL, + user_id UUID REFERENCES users(id) ON DELETE SET NULL, + key TEXT, + content JSONB NOT NULL, + text TEXT, + tags TEXT[] DEFAULT '{}', + metadata JSONB DEFAULT '{}'::jsonb, + revision INTEGER NOT NULL DEFAULT 1, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ, + deleted_at TIMESTAMPTZ +); +CREATE INDEX IF NOT EXISTS mem_items_ns_idx ON memory_items(namespace_id, thread_id, created_at DESC); +CREATE INDEX IF NOT EXISTS mem_items_tags_gin ON memory_items USING GIN(tags); +CREATE INDEX IF NOT EXISTS mem_items_meta_gin ON memory_items USING GIN(metadata); + +-- Chunks: embedding units derived from items +CREATE TABLE IF NOT EXISTS memory_chunks ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + item_id UUID NOT NULL REFERENCES memory_items(id) ON DELETE CASCADE, + ord INTEGER NOT NULL, + text TEXT NOT NULL, + metadata JSONB DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS chunks_item_idx ON memory_chunks(item_id, ord); + +-- Embeddings: one per chunk (per model) +CREATE TABLE IF NOT EXISTS embeddings ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + chunk_id UUID NOT NULL REFERENCES memory_chunks(id) ON DELETE CASCADE, + model TEXT NOT NULL, + dim INTEGER NOT NULL, + vector VECTOR(1536) NOT NULL, -- adjust dim per model + normalized BOOLEAN DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE(chunk_id, model) +); +CREATE INDEX IF NOT EXISTS embeddings_model_dim ON embeddings(model, dim); +-- For ivfflat you must first create a HNSW/IVFFLAT index; pgvector uses different syntax depending on version +CREATE INDEX IF NOT EXISTS embeddings_vector_ivfflat ON embeddings USING ivfflat (vector) WITH (lists = 100); + +-- Helper upsert function for memory_items revision bump +CREATE OR REPLACE FUNCTION bump_revision() RETURNS trigger AS $$ +BEGIN + NEW.revision := COALESCE(OLD.revision, 0) + 1; + NEW.updated_at := now(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_bump_revision ON memory_items; +CREATE TRIGGER trg_bump_revision +BEFORE UPDATE ON memory_items +FOR EACH ROW EXECUTE FUNCTION bump_revision(); diff --git a/src/dal/IDatabase.hpp b/src/dal/IDatabase.hpp new file mode 100644 index 0000000..7102ac6 --- /dev/null +++ b/src/dal/IDatabase.hpp @@ -0,0 +1,25 @@ +#pragma once +#include +#include +#include +#include "Models.hpp" + +class IDatabase { +public: + virtual ~IDatabase() = default; + virtual bool connect(const std::string& dsn) = 0; + virtual void close() = 0; + + // Transactions + virtual bool begin() = 0; + virtual bool commit() = 0; + virtual void rollback() = 0; + + // Memory ops (skeleton) + virtual std::optional ensureNamespace(const std::string& name) = 0; + virtual std::string upsertItem(const ItemRow& item) = 0; + virtual std::vector upsertChunks(const std::vector& chunks) = 0; + virtual std::vector upsertEmbeddings(const std::vector& embs) = 0; + virtual std::vector searchText(const std::string& namespace_id, const std::string& query, int k) = 0; + virtual std::vector> searchVector(const std::string& namespace_id, const std::vector& embedding, int k) = 0; +}; diff --git a/src/dal/Models.hpp b/src/dal/Models.hpp new file mode 100644 index 0000000..0f034d1 --- /dev/null +++ b/src/dal/Models.hpp @@ -0,0 +1,37 @@ +#pragma once +#include +#include +#include +#include + +struct NamespaceRow { std::string id; std::string name; }; +struct ThreadRow { std::string id; std::string namespace_id; std::string external_id; }; +struct UserRow { std::string id; std::string external_id; }; + +struct ItemRow { + std::string id; + std::string namespace_id; + std::optional thread_id; + std::optional user_id; + std::optional key; + std::string content_json; + std::optional text; + std::vector tags; + std::unordered_map metadata; + int revision{1}; +}; + +struct ChunkRow { + std::string id; + std::string item_id; + int ord{0}; + std::string text; +}; + +struct EmbeddingRow { + std::string id; + std::string chunk_id; + std::string model; + int dim{0}; + std::vector vector; +}; diff --git a/src/dal/PgDal.cpp b/src/dal/PgDal.cpp new file mode 100644 index 0000000..df071a3 --- /dev/null +++ b/src/dal/PgDal.cpp @@ -0,0 +1,22 @@ +#include "PgDal.hpp" +#include + +// NOTE: Stub implementation (no libpq linked yet). + +bool PgDal::connect(const std::string& dsn) { (void)dsn; std::cout << "[PgDal] connect stub +"; return true; } +void PgDal::close() { std::cout << "[PgDal] close stub +"; } +bool PgDal::begin() { std::cout << "[PgDal] begin stub +"; return true; } +bool PgDal::commit() { std::cout << "[PgDal] commit stub +"; return true; } +void PgDal::rollback() { std::cout << "[PgDal] rollback stub +"; } + +std::optional PgDal::ensureNamespace(const std::string& name) { (void)name; return NamespaceRow{"00000000-0000-0000-0000-000000000000", name}; } +std::string PgDal::upsertItem(const ItemRow& item) { (void)item; return std::string("00000000-0000-0000-0000-000000000001"); } +std::vector PgDal::upsertChunks(const std::vector& chunks) { return std::vector(chunks.size(), "00000000-0000-0000-0000-000000000002"); } +std::vector PgDal::upsertEmbeddings(const std::vector& embs) { return std::vector(embs.size(), "00000000-0000-0000-0000-000000000003"); } +std::vector PgDal::searchText(const std::string& namespace_id, const std::string& query, int k) { (void)namespace_id; (void)query; (void)k; return {}; } +std::vector> PgDal::searchVector(const std::string& namespace_id, const std::vector& embedding, int k) { (void)namespace_id; (void)embedding; (void)k; return {}; } diff --git a/src/dal/PgDal.hpp b/src/dal/PgDal.hpp new file mode 100644 index 0000000..3610bd2 --- /dev/null +++ b/src/dal/PgDal.hpp @@ -0,0 +1,18 @@ +#pragma once +#include "IDatabase.hpp" + +class PgDal : public IDatabase { +public: + bool connect(const std::string& dsn) override; + void close() override; + bool begin() override; + bool commit() override; + void rollback() override; + + std::optional ensureNamespace(const std::string& name) override; + std::string upsertItem(const ItemRow& item) override; + std::vector upsertChunks(const std::vector& chunks) override; + std::vector upsertEmbeddings(const std::vector& embs) override; + std::vector searchText(const std::string& namespace_id, const std::string& query, int k) override; + std::vector> searchVector(const std::string& namespace_id, const std::vector& embedding, int k) override; +}; diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 0000000..93146d9 --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,77 @@ +// Minimal CLI runner that registers Kompanion MCP tools and dispatches requests. +#include +#include +#include +#include +#include +#include "mcp/KomMcpServer.hpp" +#include "mcp/RegisterTools.hpp" + +namespace { + +std::string read_all(std::istream& in) { + std::ostringstream oss; + oss << in.rdbuf(); + return oss.str(); +} + +std::string load_request(int argc, char** argv) { + if (argc < 3) { + return read_all(std::cin); + } + std::string arg = argv[2]; + if (arg == "-") { + return read_all(std::cin); + } + std::filesystem::path p{arg}; + if (std::filesystem::exists(p)) { + std::ifstream file(p); + if (!file) throw std::runtime_error("unable to open request file: " + p.string()); + return read_all(file); + } + return arg; +} + +void print_usage(const char* exe, KomMcpServer& server) { + std::cerr << "Usage: " << exe << " [request-json|-|path]\n"; + std::cerr << "Available tools:\n"; + for (const auto& name : server.listTools()) { + std::cerr << " - " << name << "\n"; + } +} + +} // namespace + +int main(int argc, char** argv) { + KomMcpServer server; + register_default_tools(server); + + if (argc < 2) { + print_usage(argv[0], server); + return 1; + } + + std::string tool = argv[1]; + if (tool == "--list") { + for (const auto& name : server.listTools()) { + std::cout << name << "\n"; + } + return 0; + } + + if (!server.hasTool(tool)) { + std::cerr << "Unknown tool: " << tool << "\n"; + print_usage(argv[0], server); + return 1; + } + + try { + std::string request = load_request(argc, argv); + std::string response = server.dispatch(tool, request); + std::cout << response << std::endl; + return 0; + } catch (const std::exception& ex) { + std::cerr << "Error: " << ex.what() << "\n"; + return 1; + } +} diff --git a/src/mcp/HandlersLocalBackup.hpp b/src/mcp/HandlersLocalBackup.hpp new file mode 100644 index 0000000..a69b001 --- /dev/null +++ b/src/mcp/HandlersLocalBackup.hpp @@ -0,0 +1,17 @@ +#pragma once +#include + +namespace Handlers { + +inline std::string backup_export_encrypted(const std::string& reqJson) { + (void)reqJson; + // TODO: implement according to docs/backup-e2ee-spec.md + return std::string("{\"status\":\"queued\",\"artifact\":\"/tmp/export.enc\"}"); +} + +inline std::string backup_import_encrypted(const std::string& reqJson) { + (void)reqJson; + return std::string("{\"status\":\"ok\"}"); +} + +} // namespace Handlers diff --git a/src/mcp/HandlersMemory.hpp b/src/mcp/HandlersMemory.hpp new file mode 100644 index 0000000..c7827f4 --- /dev/null +++ b/src/mcp/HandlersMemory.hpp @@ -0,0 +1,104 @@ +#pragma once +#include +#include +#include + +namespace Handlers { + +inline std::string json_escape(const std::string& in) { + std::ostringstream os; + for (char c : in) { + switch (c) { + case '\"': os << "\\\""; break; + case '\\': os << "\\\\"; break; + case '\n': os << "\\n"; break; + case '\r': os << "\\r"; break; + case '\t': os << "\\t"; break; + default: os << c; break; + } + } + return os.str(); +} + +inline size_t count_items_array(const std::string& json) { + auto pos = json.find("\"items\""); + if (pos == std::string::npos) return 0; + pos = json.find('[', pos); + if (pos == std::string::npos) return 0; + size_t count = 0; + int depth = 0; + bool inString = false; + bool escape = false; + for (size_t i = pos + 1; i < json.size(); ++i) { + char c = json[i]; + if (escape) { escape = false; continue; } + if (c == '\\') { if (inString) escape = true; continue; } + if (c == '\"') { inString = !inString; continue; } + if (inString) continue; + if (c == '{') { + if (depth == 0) ++count; + ++depth; + } else if (c == '}') { + if (depth > 0) --depth; + } else if (c == ']' && depth == 0) { + break; + } + } + return count; +} + +inline std::string extract_string_field(const std::string& json, const std::string& key) { + const std::string pattern = "\"" + key + "\""; + auto pos = json.find(pattern); + if (pos == std::string::npos) return {}; + pos = json.find(':', pos); + if (pos == std::string::npos) return {}; + ++pos; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) ++pos; + if (pos >= json.size() || json[pos] != '\"') return {}; + ++pos; + std::ostringstream os; + bool escape = false; + for (; pos < json.size(); ++pos) { + char c = json[pos]; + if (escape) { + switch (c) { + case '\"': os << '\"'; break; + case '\\': os << '\\'; break; + case '/': os << '/'; break; + case 'b': os << '\b'; break; + case 'f': os << '\f'; break; + case 'n': os << '\n'; break; + case 'r': os << '\r'; break; + case 't': os << '\t'; break; + default: os << c; break; + } + escape = false; + continue; + } + if (c == '\\') { escape = true; continue; } + if (c == '\"') break; + os << c; + } + return os.str(); +} + +inline std::string upsert_memory(const std::string& reqJson) { + size_t count = count_items_array(reqJson); + std::ostringstream os; + os << "{\"upserted\":" << count << ",\"status\":\"ok\"}"; + return os.str(); +} + +inline std::string search_memory(const std::string& reqJson) { + std::string queryText = extract_string_field(reqJson, "text"); + std::ostringstream os; + os << "{\"matches\":["; + if (!queryText.empty()) { + os << "{\"id\":\"stub-memory-1\",\"score\":0.42,\"text\":\"" << json_escape(queryText) << "\"}"; + } + os << "]}"; + return os.str(); +} + +} // namespace Handlers diff --git a/src/mcp/KomMcpServer.hpp b/src/mcp/KomMcpServer.hpp new file mode 100644 index 0000000..0959199 --- /dev/null +++ b/src/mcp/KomMcpServer.hpp @@ -0,0 +1,39 @@ +#pragma once +#include +#include +#include +#include + +// Minimal, dependency-free server facade we can later back with qtmcp. +// Handlers take JSON strings (request) and return JSON strings (response). +class KomMcpServer { +public: + using Handler = std::function; + + void registerTool(const std::string& name, Handler handler) { + tools_[name] = std::move(handler); + } + + bool hasTool(const std::string& name) const { + return tools_.count(name) > 0; + } + + std::vector listTools() const { + std::vector out; + out.reserve(tools_.size()); + for (auto &kv : tools_) out.push_back(kv.first); + return out; + } + + // For now, just dispatch synchronously. + std::string dispatch(const std::string& name, const std::string& requestJson) { + if (!hasTool(name)) return std::string("{\"error\":{\"code\":\"tool_not_found\",\"message\":\""+name+"\"}}\n"); + return tools_[name](requestJson); + } + + // Placeholder for actual MCP run loop (qtmcp). + int runOnce() { return 0; } + +private: + std::unordered_map tools_; +}; diff --git a/src/mcp/RateLimiter.hpp b/src/mcp/RateLimiter.hpp new file mode 100644 index 0000000..39a1aba --- /dev/null +++ b/src/mcp/RateLimiter.hpp @@ -0,0 +1,24 @@ +#pragma once +#include +#include +#include + +class RateLimiter { +public: + // simple fixed window per tool + bool allow(const std::string& tool, size_t maxPerWindow = 5, std::chrono::seconds window = std::chrono::seconds(1)) { + using clock = std::chrono::steady_clock; + auto now = clock::now(); + auto &state = buckets_[tool]; + if (now - state.windowStart > window) { + state.windowStart = now; + state.count = 0; + } + if (state.count >= maxPerWindow) return false; + ++state.count; + return true; + } +private: + struct Bucket { std::chrono::steady_clock::time_point windowStart{}; size_t count{0}; }; + std::unordered_map buckets_; +}; diff --git a/src/mcp/RegisterTools.hpp b/src/mcp/RegisterTools.hpp new file mode 100644 index 0000000..3a5e7d0 --- /dev/null +++ b/src/mcp/RegisterTools.hpp @@ -0,0 +1,11 @@ +#pragma once +#include "KomMcpServer.hpp" +#include "HandlersLocalBackup.hpp" +#include "HandlersMemory.hpp" + +inline void register_default_tools(KomMcpServer& server) { + server.registerTool("kom.memory.v1.upsert_memory", Handlers::upsert_memory); + server.registerTool("kom.memory.v1.search_memory", Handlers::search_memory); + server.registerTool("kom.local.v1.backup.export_encrypted", Handlers::backup_export_encrypted); + server.registerTool("kom.local.v1.backup.import_encrypted", Handlers::backup_import_encrypted); +} diff --git a/src/mcp/Scope.hpp b/src/mcp/Scope.hpp new file mode 100644 index 0000000..235d95b --- /dev/null +++ b/src/mcp/Scope.hpp @@ -0,0 +1,14 @@ +#pragma once +#include + +struct Scope { + std::string namespace_id; + std::string user_id; + bool valid() const { return !namespace_id.empty(); } +}; + +// In a real qtmcp server, these would be derived from session/auth. +inline Scope currentScope() { + // TODO: inject from session context / env. For now, return empty (invalid). + return Scope{}; +} diff --git a/src/mcp/ToolSchemas.json b/src/mcp/ToolSchemas.json new file mode 100644 index 0000000..8c287ae --- /dev/null +++ b/src/mcp/ToolSchemas.json @@ -0,0 +1,196 @@ +{ + "namespace": "kom.memory.v1", + "tools": { + "save_context": { + "input": { + "type": "object", + "properties": { + "namespace": {"type": "string"}, + "key": {"type": "string"}, + "content": {}, + "tags": {"type": "array", "items": {"type": "string"}}, + "ttl_seconds": {"type": "integer"} + }, + "required": ["namespace", "content"] + }, + "output": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "created_at": {"type": "string"} + }, + "required": ["id", "created_at"] + } + }, + "recall_context": { + "input": { + "type": "object", + "properties": { + "namespace": {"type": "string"}, + "key": {"type": "string"}, + "tags": {"type": "array", "items": {"type": "string"}}, + "limit": {"type": "integer"}, + "since": {"type": "string"} + }, + "required": ["namespace"] + }, + "output": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "key": {"type": "string"}, + "content": {}, + "tags": {"type": "array", "items": {"type": "string"}}, + "created_at": {"type": "string"} + }, + "required": ["id", "content", "created_at"] + } + } + }, + "required": ["items"] + } + }, + "embed_text": { + "input": { + "type": "object", + "properties": { + "model": {"type": "string"}, + "texts": {"type": "array", "items": {"type": "string"}} + }, + "required": ["texts"] + }, + "output": { + "type": "object", + "properties": { + "model": {"type": "string"}, + "vectors": {"type": "array", "items": {"type": "array", "items": {"type": "number"}}} + }, + "required": ["model", "vectors"] + } + }, + "upsert_memory": { + "input": { + "type": "object", + "properties": { + "namespace": {"type": "string"}, + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "text": {"type": "string"}, + "metadata": {"type": "object"}, + "embedding": {"type": "array", "items": {"type": "number"}} + }, + "required": ["text"] + } + } + }, + "required": ["namespace", "items"] + }, + "output": { + "type": "object", + "properties": { + "upserted": {"type": "integer"} + }, + "required": ["upserted"] + } + }, + "search_memory": { + "input": { + "type": "object", + "properties": { + "namespace": {"type": "string"}, + "query": { + "type": "object", + "properties": { + "text": {"type": "string"}, + "embedding": {"type": "array", "items": {"type": "number"}}, + "k": {"type": "integer"}, + "filter": {"type": "object"} + } + } + }, + "required": ["namespace", "query"] + }, + "output": { + "type": "object", + "properties": { + "matches": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "score": {"type": "number"}, + "text": {"type": "string"}, + "metadata": {"type": "object"} + }, + "required": ["id", "score"] + } + } + }, + "required": ["matches"] + } + }, + "warm_cache": { + "input": { + "type": "object", + "properties": { + "namespace": {"type": "string"}, + "since": {"type": "string"} + }, + "required": ["namespace"] + }, + "output": { + "type": "object", + "properties": { + "queued": {"type": "integer"} + }, + "required": ["queued"] + } + }, + + "kom.local.v1.backup.export_encrypted": { + "input": { + "type": "object", + "properties": { + "namespaces": {"type": "array", "items": {"type": "string"}}, + "destination": {"type": "string"} + }, + "required": ["namespaces", "destination"] + }, + "output": { + "type": "object", + "properties": { + "status": {"type": "string"}, + "artifact": {"type": "string"} + }, + "required": ["status", "artifact"] + } + }, + + "kom.local.v1.backup.import_encrypted": { + "input": { + "type": "object", + "properties": { + "source": {"type": "string"} + }, + "required": ["source"] + }, + "output": { + "type": "object", + "properties": { + "status": {"type": "string"} + }, + "required": ["status"] + } + } + } +} diff --git a/src/mcp/Tools.hpp b/src/mcp/Tools.hpp new file mode 100644 index 0000000..8f53366 --- /dev/null +++ b/src/mcp/Tools.hpp @@ -0,0 +1,40 @@ +#pragma once +#include +#include +#include + +namespace Tools { + +// Super-lightweight JSON helpers (NOT robust; only for stub/testing). +inline std::string json_kv(const std::string& k, const std::string& v) { + std::ostringstream os; os << "\"" << k << "\"\: \"" << v << "\""; return os.str(); +} +inline std::string json_kv_num(const std::string& k, double v) { + std::ostringstream os; os << "\"" << k << "\"\: " << v; return os.str(); +} +inline std::string json_arr(const std::vector& items) { + std::ostringstream os; os << "["; + for (size_t i=0;i& toolNames) { + std::vector quoted; quoted.reserve(toolNames.size()); + for (auto &t: toolNames) { std::ostringstream q; q << "\"" << t << "\""; quoted.push_back(q.str()); } + std::ostringstream os; + os << "{" << json_kv("status", "ok") << ", " + << "\"tools\": " << json_arr(quoted) << "}"; + return os.str(); +} + +// `embed_text` stub: returns zero vectors of dimension 8 for each input text +inline std::string embed_text_stub(size_t n) { + std::ostringstream os; + os << "{\"model\":\"stub-embed-8d\",\"vectors\":["; + for (size_t i=0;i +#include +#include + +struct EmbedResult { + std::string model; + std::vector> vectors; +}; + +class IEmbedder { +public: + virtual ~IEmbedder() = default; + virtual EmbedResult embed(const std::vector& texts, std::optional model = std::nullopt) = 0; +}; diff --git a/src/memory/IVectorStore.hpp b/src/memory/IVectorStore.hpp new file mode 100644 index 0000000..09fdf86 --- /dev/null +++ b/src/memory/IVectorStore.hpp @@ -0,0 +1,30 @@ +#pragma once +#include +#include +#include +#include + +struct MemoryItem { + std::string id; + std::string text; + std::unordered_map metadata; +}; + +struct QueryFilter { + std::unordered_map equals; +}; + +struct QueryResult { + std::string id; + float score; + std::optional text; + std::unordered_map metadata; +}; + +class IVectorStore { +public: + virtual ~IVectorStore() = default; + virtual size_t upsert(const std::string& nameSpace, const std::vector& items, const std::vector>* embeddings = nullptr) = 0; + virtual std::vector query(const std::string& nameSpace, const std::vector& embedding, size_t k = 8, std::optional filter = std::nullopt) = 0; + virtual bool remove(const std::string& nameSpace, const std::vector& ids) = 0; +}; diff --git a/tasks-table.md b/tasks-table.md new file mode 100644 index 0000000..0844c78 --- /dev/null +++ b/tasks-table.md @@ -0,0 +1,43 @@ +# metal-kompanion-mcp +MCP backend for Kompanion: memory/context/embedding provider over MCP, built from scratch (qtmcp-based) to persist conversation state and serve embeddings + retrieval to avoid forgetting across threads. + +> ## 📈 Project Summary +> +> **✅ Done**: 0 | **🔄 In Progress**: 0 | **⬜ Todo**: 19 | **❌ Blocked**: 0 +> +> **Progress**: 0% `░░░░░░░░░░░░░░░░░░░░` 0/19 tasks +> +> **Priorities**: 🚨 **Critical**: 0 | 🔴 **High**: 1 | 🟡 **Medium**: 20 | 🟢 **Low**: 0 + +## Tasks + +| ID | Status | Priority | Title | Description | +|:--:|:------:|:--------:|:------|:------------| +| #1 | ⬜ todo | 700 | **Project Setup: metal-kompanion-mcp** | MCP backend for Kompanion: me... | +| #2 | ⬜ in_progress | 500 | **Design MCP memory/context API** | Specify MCP tools for: save_c... | +| #3 | ⬜ todo | 501 | **Select embedding backend & storage** | Choose between local (Ollama/... | +| #4 | ⬜ in_progress | 499 | **Scaffold qtmcp-based server** | Set up C++/Qt MCP server skel... | +| #5 | ⬜ todo | 502 | **Implement memory adapters** | Adapters: (1) SQLite+FAISS/pg... | +| #6 | ⬜ todo | 498 | **Deep research: memory DB architecture & schema** | Survey best practices for con... | +| #7 | ⬜ todo | 503 | **Decide primary DB: Postgres+pgvector vs SQLite+FAISS** | Evaluate tradeoffs (multi-use... | +| #8 | ⬜ todo | 497 | **Implement DAL + migrations (pgvector)** | Create C++ DAL layer for name... | +| #9 | ⬜ todo | 504 | **Add cloud DB hardening (RLS, FTS/trgm, ANN indexes)** | Implement RLS policies; add F... | +| #10 | ⬜ todo | 496 | **Server enforcement: scope injection + rate limits** | Inject namespace/user via ses... | +| #11 | ⬜ todo | 505 | **Redaction & sensitivity pipeline** | Implement preprocessing to de... | +| #12 | ⬜ todo | 495 | **Private vault mode (key-only retrieval)** | Implement vault path for secr... | +| #13 | ⬜ todo | 506 | **Local backup tools: export/import (E2EE)** | Add kom.local.v1.backup.expor... | +| #14 | ⬜ todo | 494 | **Cloud adapters: backup/sync & payments stubs** | Expose kom.cloud.v1.backup.up... | +| #15 | ⬜ todo | 507 | **Purge job & admin delete paths** | Implement scheduled hard-dele... | +| #16 | ⬜ todo | 493 | **Test suite: privacy & hybrid search** | Cross-tenant leakage, redacti... | +| #17 | ⬜ todo | 508 | **Enable Qwen-2.5-Coder with tool support (Happy-Code profile)** | Prepare system prompt + regis... | +| #18 | ⬜ todo | 492 | **Expose Agentic-Control-Framework as a tool** | Wrap ACF endpoints into a too... | +| #19 | ⬜ todo | 509 | **DAL skeleton + SQL calls (pgvector)** | Create DAL interfaces and pgv... | +| #20 | ⬜ todo | 491 | **Claude Code integration rescue plan** | Stabilize Qwen-2.5-Coder insi... | + + +### Task #2: Design MCP memory/context API - Subtasks + +| ID | Status | Title | +|:--:|:------:|:------| +| #2.1 | ⬜ todo | Write JSON Schemas for tools (done) | + diff --git a/tests/contract/test_mcp_tools.cpp b/tests/contract/test_mcp_tools.cpp new file mode 100644 index 0000000..0fbbbec --- /dev/null +++ b/tests/contract/test_mcp_tools.cpp @@ -0,0 +1,43 @@ +#include +#include +#include "mcp/KomMcpServer.hpp" +#include "mcp/RegisterTools.hpp" + +namespace { + +bool expect_contains(const std::string& haystack, const std::string& needle, const std::string& context) { + if (haystack.find(needle) == std::string::npos) { + std::cerr << "Expected response to contain '" << needle << "' but got:\n" + << haystack << "\nContext: " << context << "\n"; + return false; + } + return true; +} + +} // namespace + +int main() { + KomMcpServer server; + register_default_tools(server); + + const std::string upsertReq = R"({"namespace":"tests","items":[{"text":"hello world"},{"text":"hola mundo"}]})"; + std::string upsertResp = server.dispatch("kom.memory.v1.upsert_memory", upsertReq); + if (!expect_contains(upsertResp, "\"upserted\":2", "upsert_memory count")) return 1; + if (!expect_contains(upsertResp, "\"status\":\"ok\"", "upsert_memory status")) return 1; + + const std::string searchReq = R"({"namespace":"tests","query":{"text":"hello","k":3}})"; + std::string searchResp = server.dispatch("kom.memory.v1.search_memory", searchReq); + if (!expect_contains(searchResp, "\"matches\"", "search_memory matches key")) return 1; + if (!expect_contains(searchResp, "\"text\":\"hello\"", "search_memory echo text")) return 1; + + const std::string exportReq = R"({"namespace":"tests","destination":"/tmp/example.enc"})"; + std::string exportResp = server.dispatch("kom.local.v1.backup.export_encrypted", exportReq); + if (!expect_contains(exportResp, "\"status\":\"queued\"", "export status")) return 1; + if (!expect_contains(exportResp, "\"artifact\"", "export artifact path")) return 1; + + const std::string importReq = R"({"namespace":"tests","source":"/tmp/example.enc"})"; + std::string importResp = server.dispatch("kom.local.v1.backup.import_encrypted", importReq); + if (!expect_contains(importResp, "\"status\":\"ok\"", "import status")) return 1; + + return 0; +} diff --git a/tests/schemas/sample_search.json b/tests/schemas/sample_search.json new file mode 100644 index 0000000..addad9f --- /dev/null +++ b/tests/schemas/sample_search.json @@ -0,0 +1,10 @@ +{ + "namespace": "project:metal", + "query": { + "text": "embedding model", + "k": 5, + "filter": { + "thread": "embedding-comparison" + } + } +} diff --git a/tests/schemas/sample_upsert.json b/tests/schemas/sample_upsert.json new file mode 100644 index 0000000..2ed9fa8 --- /dev/null +++ b/tests/schemas/sample_upsert.json @@ -0,0 +1,6 @@ +{ + "namespace": "project:metal", + "items": [ + {"id": "note-1", "text": "Embedding model comparison: takeaways...", "metadata": {"thread": "embedding-comparison"}} + ] +} diff --git a/tools/ingest_dir.py b/tools/ingest_dir.py new file mode 100755 index 0000000..5eb9009 --- /dev/null +++ b/tools/ingest_dir.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +import os, sys, hashlib, psycopg, requests, json + +DB=os.environ.get("DB_URL","dbname=kompanion user=kompanion host=/var/run/postgresq") +OLLAMA=os.environ.get("OLLAMA_BASE","http://127.0.0.1:11434") +MODEL=os.environ.get("EMBED_MODEL","mxbai-embed-large") +SPACE=os.environ.get("EMBED_SPACE","dev_knowledge") # dev_knowledge | pattern_exchange | runtime_memory + +def sha256(p): + h=hashlib.sha256() + with open(p,"rb") as f: + for chunk in iter(lambda: f.read(1<<20), b""): h.update(chunk) + return h.hexdigest() + +def embed(text): + r=requests.post(f"{OLLAMA}/api/embeddings", json={"model": MODEL, "prompt": text}, timeout=120) + r.raise_for_status(); return r.json()["embedding"] + +def chunks(s, sz=1600): + b=s.encode("utf-8"); + for i in range(0,len(b),sz): yield b[i:i+sz].decode("utf-8","ignore") + +def insert_embedding(cur, dim, kid, sid, vec): + if dim==768: + cur.execute("INSERT INTO komp.embedding_768(chunk_id,space_id,embedding) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING",(kid,sid,vec)) + elif dim==1024: + cur.execute("INSERT INTO komp.embedding_1024(chunk_id,space_id,embedding) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING",(kid,sid,vec)) + else: + return False + return True + +def main(root): + with psycopg.connect(DB) as conn, conn.cursor() as cur: + cur.execute("SELECT id,dim FROM komp.space WHERE name=%s",(SPACE,)) + row=cur.fetchone() + if not row: raise SystemExit(f"space {SPACE} missing (init schema)") + sid, target_dim = row + for dirpath,_,files in os.walk(root): + for fn in files: + p=os.path.join(dirpath,fn) + if os.path.getsize(p)==0: continue + # include common text/code; PDFs via pdftotext if available + if not any(fn.lower().endswith(ext) for ext in (".md",".txt",".json",".py",".cpp",".c",".hpp",".yaml",".yml",".toml",".pdf",".mdown",".rst",".org",".js",".ts",".sql",".sh",".ini",".conf",".cfg",".log",".mime")): + continue + if fn.lower().endswith(".pdf"): + try: + txt=os.popen(f"pdftotext -layout -nopgbrk '{p}' - -q").read() + except Exception: + continue + else: + try: txt=open(p,"r",encoding="utf-8",errors="ignore").read() + except Exception: continue + sh=sha256(p) + cur.execute("INSERT INTO komp.source(kind,uri,meta) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING RETURNING id", + ("filesystem",p,json.dumps({}))) + sid_src = cur.fetchone()[0] if cur.rowcount else None + if not sid_src: + cur.execute("SELECT id FROM komp.source WHERE kind='filesystem' AND uri=%s",(p,)) + sid_src=cur.fetchone()[0] + ln=1 + for ch in chunks(txt): + cur.execute("INSERT INTO komp.chunk(source_id,lineno,text,sha256,tokens) VALUES(%s,%s,%s,%s,%s) RETURNING id", + (sid_src,ln,ch,sh,len(ch)//4)) + kid=cur.fetchone()[0] + vec=embed(ch) + if len(vec)!=target_dim: + cur.execute("DELETE FROM komp.chunk WHERE id=%s",(kid,)) + else: + insert_embedding(cur, target_dim, kid, sid, vec) + ln += ch.count("\n")+1 + conn.commit() + print("done") + +if __name__=='__main__': + if len(sys.argv)<2: print("usage: ingest_dir.py [space]", file=sys.stderr); sys.exit(1) + if len(sys.argv)>=3: os.environ["EMBED_SPACE"]=sys.argv[2] + main(sys.argv[1]) + diff --git a/tools/pg_search.py b/tools/pg_search.py new file mode 100644 index 0000000..28faf9c --- /dev/null +++ b/tools/pg_search.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +import os, sys, json, requests, psycopg + +DB=os.environ.get("DB_URL","dbname=kompanion user=kompanion host=/var/run/postgresql") +OLLAMA=os.environ.get("OLLAMA_BASE","http://127.0.0.1:11434") +MODEL=os.environ.get("EMBED_MODEL","mxbai-embed-large") +SPACE=os.environ.get("EMBED_SPACE","dev_knowledge") + +HELP="""\ +Usage: pg_search.py "query text" [k] +Env: DB_URL, OLLAMA_BASE, EMBED_MODEL, EMBED_SPACE (default dev_knowledge) +Prints JSON results: [{score, uri, lineno, text}]. +""" + +def embed(q: str): + r = requests.post(f"{OLLAMA}/api/embeddings", json={"model": MODEL, "prompt": q}, timeout=120) + r.raise_for_status() + return r.json()["embedding"] + +if __name__=="__main__": + if len(sys.argv)<2: + print(HELP, file=sys.stderr); sys.exit(1) + query = sys.argv[1] + k = int(sys.argv[2]) if len(sys.argv)>2 else 8 + vec = embed(query) + with psycopg.connect(DB) as conn, conn.cursor() as cur: + cur.execute("SELECT id, dim FROM komp.space WHERE name=%s", (SPACE,)) + row = cur.fetchone() + if not row: + sys.exit(f"space {SPACE} missing") + sid, dim = row + if dim not in (768,1024): + sys.exit(f"unsupported dim {dim}") + table = f"komp.embedding_{dim}" + # cosine distance with vector_cosine_ops + sql = f""" + SELECT (e.embedding <=> %(v)s::vector) AS score, s.uri, k.lineno, k.text + FROM {table} e + JOIN komp.chunk k ON k.id = e.chunk_id + JOIN komp.source s ON s.id = k.source_id + WHERE e.space_id = %(sid)s + ORDER BY e.embedding <=> %(v)s::vector + LIMIT %(k)s + """ + cur.execute(sql, {"v": vec, "sid": sid, "k": k}) + out=[{"score":float(r[0]),"uri":r[1],"lineno":r[2],"text":r[3]} for r in cur.fetchall()] + print(json.dumps(out, ensure_ascii=False, indent=2)) + diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 0000000..296ca87 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1 @@ +psycopg