From c7e55cfd24f2ac8a7c0439bf1c1e61827234cb54 Mon Sep 17 00:00:00 2001 From: admin Date: Wed, 27 May 2026 10:49:05 +0900 Subject: [PATCH] feat: add Qdrant recording workflow --- .gitea/workflows/record-to-qdrant.yml | 114 ++++++++++++++++++++++++++ scripts/build_digest_payload.py | 22 +++++ scripts/build_embed_input.py | 3 + scripts/build_gen_payload.py | 6 ++ scripts/extract_texts.py | 4 + scripts/record_event.py | 25 ++++++ 6 files changed, 174 insertions(+) create mode 100644 .gitea/workflows/record-to-qdrant.yml create mode 100644 scripts/build_digest_payload.py create mode 100644 scripts/build_embed_input.py create mode 100644 scripts/build_gen_payload.py create mode 100644 scripts/extract_texts.py create mode 100644 scripts/record_event.py diff --git a/.gitea/workflows/record-to-qdrant.yml b/.gitea/workflows/record-to-qdrant.yml new file mode 100644 index 0000000..f2c5298 --- /dev/null +++ b/.gitea/workflows/record-to-qdrant.yml @@ -0,0 +1,114 @@ +name: Record to Qdrant + +on: + push: + branches: ['**'] + pull_request: + types: [opened, closed, synchronize] + create: + delete: + release: + types: [published] + workflow_dispatch: + schedule: + - cron: '0 15 * * *' + +jobs: + record-event: + if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' + runs-on: ubuntu-latest + container: + image: python:3.11-slim + defaults: + run: + shell: bash + steps: + - name: Install dependencies + run: apt-get update -qq && apt-get install -y -qq curl bash git + + - name: Checkout + run: | + git clone --depth 1 http://admin:0ebeb908eb6bc62c20d4d0cb937970651859f409@10.43.165.66:3000/k3s-manifests/pod-manifests.git /tmp/repo + + - name: Record event to Qdrant + env: + EVENT: ${{ github.event_name }} + REPO: ${{ github.repository }} + BRANCH: ${{ github.ref_name }} + ACTOR: ${{ github.actor }} + RUN_ID: ${{ github.run_id }} + COMMIT: ${{ github.sha }} + run: | + COMMIT_SHORT=$(echo "${COMMIT}" | cut -c1-8) + TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + CONTENT="[${EVENT}] ${REPO} @ ${BRANCH} by ${ACTOR} | commit: ${COMMIT_SHORT} | run: ${RUN_ID} | time: ${TIMESTAMP}" + echo "Content: ${CONTENT}" + echo -n "${CONTENT}" > /tmp/content.txt + + python3 -c "import json;open('/tmp/embed_req.json','w').write(json.dumps({'model':'bge-m3:latest','input':open('/tmp/content.txt').read()}))" + + curl -s -X POST http://192.168.11.124:11434/api/embed \ + -H "Content-Type: application/json" \ + --data-binary @/tmp/embed_req.json > /tmp/embed_response.json + + export TIMESTAMP RUN_ID EVENT REPO BRANCH ACTOR + python3 /tmp/repo/scripts/record_event.py + + curl -s -X PUT http://192.168.11.124:6333/collections/gitea_actions/points \ + -H "Content-Type: application/json" \ + --data-binary @/tmp/payload.json + echo "Done" + + nightly-digest: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + container: + image: python:3.11-slim + defaults: + run: + shell: bash + steps: + - name: Install dependencies + run: apt-get update -qq && apt-get install -y -qq curl bash git + + - name: Checkout + run: | + git clone --depth 1 http://admin:0ebeb908eb6bc62c20d4d0cb937970651859f409@10.43.165.66:3000/k3s-manifests/pod-manifests.git /tmp/repo + + - name: Summarize and store digest + run: | + TODAY=$(date -u +"%Y-%m-%d") + TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + curl -s -X POST http://192.168.11.124:6333/collections/gitea_actions/points/scroll \ + -H "Content-Type: application/json" \ + --data-binary "{\"limit\":100,\"with_payload\":true,\"with_vector\":false,\"filter\":{\"must\":[{\"key\":\"timestamp\",\"range\":{\"gte\":\"${TODAY}T00:00:00Z\"}}]}}" \ + > /tmp/scroll.json + + COUNT=$(python3 -c "import json;d=json.load(open('/tmp/scroll.json'));print(len(d.get('result',{}).get('points',[])))") + echo "Records today: ${COUNT}" + if [ "${COUNT}" = "0" ]; then echo "No records, skipping."; exit 0; fi + + python3 /tmp/repo/scripts/extract_texts.py + + TODAY=${TODAY} python3 /tmp/repo/scripts/build_gen_payload.py + + curl -s -X POST http://192.168.11.124:11434/api/generate \ + -H "Content-Type: application/json" \ + --data-binary @/tmp/gen_payload.json > /tmp/gen_response.json + + python3 -c "import json;open('/tmp/summary.txt','w').write(json.load(open('/tmp/gen_response.json')).get('response',''))" + echo "Summary: $(cat /tmp/summary.txt | head -c 100)" + + python3 /tmp/repo/scripts/build_embed_input.py + + curl -s -X POST http://192.168.11.124:11434/api/embed \ + -H "Content-Type: application/json" \ + --data-binary @/tmp/embed_input.json > /tmp/embed_response.json + + TODAY=${TODAY} TIMESTAMP=${TIMESTAMP} COUNT=${COUNT} python3 /tmp/repo/scripts/build_digest_payload.py + + RESULT=$(curl -s -X PUT http://192.168.11.124:6333/collections/llm_conversations_digest/points \ + -H "Content-Type: application/json" \ + --data-binary @/tmp/digest_payload.json) + echo "Digest result: ${RESULT}" diff --git a/scripts/build_digest_payload.py b/scripts/build_digest_payload.py new file mode 100644 index 0000000..40b3d66 --- /dev/null +++ b/scripts/build_digest_payload.py @@ -0,0 +1,22 @@ +import json, uuid, os + +summary = open('/tmp/summary.txt').read() +vector = json.load(open('/tmp/embed_response.json'))['embeddings'][0] +point_id = str(uuid.uuid4()) + +payload = { + "points": [{ + "id": point_id, + "vector": vector, + "payload": { + "date": os.environ.get('TODAY', ''), + "type": "gitea_daily_digest", + "record_count": int(os.environ.get('COUNT', '0')), + "timestamp": os.environ.get('TIMESTAMP', ''), + "content": summary + } + }] +} +with open('/tmp/digest_payload.json', 'w') as f: + json.dump(payload, f) +print(f"Digest Point ID: {point_id}") diff --git a/scripts/build_embed_input.py b/scripts/build_embed_input.py new file mode 100644 index 0000000..51d327b --- /dev/null +++ b/scripts/build_embed_input.py @@ -0,0 +1,3 @@ +import json +summary = open('/tmp/summary.txt').read() +open('/tmp/embed_input.json','w').write(json.dumps({'model':'bge-m3:latest','input':summary})) diff --git a/scripts/build_gen_payload.py b/scripts/build_gen_payload.py new file mode 100644 index 0000000..761dfa2 --- /dev/null +++ b/scripts/build_gen_payload.py @@ -0,0 +1,6 @@ +import json, sys, os +today = os.environ.get('TODAY', '') +prompt = open('/tmp/texts.txt').read() +full_prompt = f"以下は{today}のGitea操作ログです。重要な変更点と傾向を日本語300字以内でまとめてください:\n{prompt}" +payload = {'model': 'qwen2.5:7b', 'prompt': full_prompt, 'stream': False} +open('/tmp/gen_payload.json','w').write(json.dumps(payload)) diff --git a/scripts/extract_texts.py b/scripts/extract_texts.py new file mode 100644 index 0000000..5b822b4 --- /dev/null +++ b/scripts/extract_texts.py @@ -0,0 +1,4 @@ +import json +d = json.load(open('/tmp/scroll.json')) +texts = '\n'.join(p.get('payload',{}).get('content','') for p in d.get('result',{}).get('points',[])) +open('/tmp/texts.txt','w').write(texts) diff --git a/scripts/record_event.py b/scripts/record_event.py new file mode 100644 index 0000000..fb5d8ae --- /dev/null +++ b/scripts/record_event.py @@ -0,0 +1,25 @@ +import json, uuid, sys, os + +content = open('/tmp/content.txt').read() +embed_resp = json.load(open('/tmp/embed_response.json')) +vector = embed_resp['embeddings'][0] +point_id = str(uuid.uuid4()) + +payload = { + "points": [{ + "id": point_id, + "vector": vector, + "payload": { + "run_id": os.environ.get('RUN_ID', ''), + "event": os.environ.get('EVENT', ''), + "repository": os.environ.get('REPO', ''), + "branch": os.environ.get('BRANCH', ''), + "actor": os.environ.get('ACTOR', ''), + "timestamp": os.environ.get('TIMESTAMP', ''), + "content": content + } + }] +} +with open('/tmp/payload.json', 'w') as f: + json.dump(payload, f) +print(f"Point ID: {point_id}")