#!/usr/bin/env bash
# ingest.sh - Load the synthetic breach dataset into Elasticsearch
# Run inside the elk-tools container or from the host after the stack is up.
# Usage: bash ingest.sh

set -euo pipefail

ES_HOST="${ES_HOST:-http://elasticsearch:9200}"
INDEX="ch18-breach"

echo "[*] Waiting for Elasticsearch at ${ES_HOST} ..."
until curl -s -o /dev/null -w "%{http_code}" "${ES_HOST}/_cluster/health" | grep -qE "^2"; do
  sleep 3
done
echo "[+] Elasticsearch is up."

echo "[*] Creating index ${INDEX} ..."
curl -s -X PUT "${ES_HOST}/${INDEX}" \
  -H "Content-Type: application/json" \
  -d '{"settings":{"number_of_shards":1,"number_of_replicas":0}}' | python3 -m json.tool

echo "[*] Ingesting breach-dataset.ndjson ..."
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Build NDJSON bulk format: each doc needs an action line
python3 - <<'PYEOF'
import json, sys, os

data_file = os.path.join(os.environ.get("SCRIPT_DIR", "."), "breach-dataset.ndjson")
with open(data_file) as f:
    lines = [l.strip() for l in f if l.strip() and not l.startswith("#")]

bulk = []
for line in lines:
    bulk.append('{"index":{}}')
    bulk.append(line)

print("\n".join(bulk))
PYEOF

# Re-run properly with the variable
python3 - "${SCRIPT_DIR}/breach-dataset.ndjson" <<'PYEOF'
import json, sys

data_file = sys.argv[1]
with open(data_file) as f:
    lines = [l.strip() for l in f if l.strip() and not l.startswith("#")]

bulk = "\n".join(
    item
    for line in lines
    for item in ['{"index":{}}', line]
) + "\n"

with open("/tmp/bulk.ndjson", "w") as out:
    out.write(bulk)
print(f"[*] Prepared {len(lines)} documents.")
PYEOF

curl -s -X POST "${ES_HOST}/${INDEX}/_bulk" \
  -H "Content-Type: application/x-ndjson" \
  --data-binary @/tmp/bulk.ndjson | python3 -m json.tool | grep -E "errors|took"

echo "[+] Ingest complete. Index: ${INDEX}"
echo "    Open Kibana -> Stack Management -> Data Views -> Create data view -> pattern: ch18-breach*"
