Confluent event configuration tutorial

INTEGRATION REQUEST — Confluent Cloud / Kafka (Real-Time Events)
================================================================

Org ID:              TN-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
Environment:         Production / Sandbox
Requester Name:      Your Name
Contact Email:       [email protected]

Kafka Topic Name:    pismo-events
Bootstrap Server:    pkc-xxxxx.us-east-1.aws.confluent.cloud:9092
API Key:             ABCDE12345FGHIJ67890
Token File Name:     confluent-api-secret.txt
  (API Secret is stored in this .txt file in the root of your
   file integration bucket — DO NOT paste the secret here)

# Log in
confluent login

# List environments and select one
confluent environment list
confluent environment use <env-id>

# List clusters and select one
confluent kafka cluster list
confluent kafka cluster use <cluster-id>

# Step 1 — Create topic
confluent kafka topic create pismo-events --partitions 6

# Step 2 — Create API key
confluent api-key create --resource <cluster-id> --description "Pismo Data Platform"
# ⚠️ Note the API Key and Secret from the output
# Store the Secret in a .txt file — never paste in tickets

# Verify topic and key
confluent kafka topic list
confluent api-key list --resource <cluster-id>

confluent kafka cluster describe --output json | jq -r '.endpoint'
# Or in Confluent Cloud Console → Cluster → Settings → Endpoints

#!/bin/bash
# ============================================================
#  Confluent Cloud Troubleshooting — Pismo Data Platform
# ============================================================
TOPIC="pismo-events"                                              # ← replace
BOOTSTRAP="pkc-xxxxx.us-east-1.aws.confluent.cloud:9092"         # ← replace
PASS=0; FAIL=0

echo "============================================================"
echo "  Confluent Cloud Integration Diagnostics"
echo "============================================================"

# 1. Confluent CLI authenticated?
echo "[1/5] Checking Confluent CLI..."
if confluent kafka cluster list >/dev/null 2>&1; then
    echo "  ✅ PASS — Authenticated and cluster accessible"
    PASS=$((PASS+1))
else
    echo "  ❌ FAIL — Not authenticated or no cluster selected"
    echo "  → confluent login && confluent environment use <env> && confluent kafka cluster use <cluster>"
    FAIL=$((FAIL+1))
fi

# 2. Topic exists?
echo "[2/5] Checking topic..."
if confluent kafka topic describe "$TOPIC" >/dev/null 2>&1; then
    echo "  ✅ PASS — Topic $TOPIC exists"
    PASS=$((PASS+1))
else
    echo "  ❌ FAIL — Topic $TOPIC not found"
    echo "  → confluent kafka topic create $TOPIC --partitions 6"
    FAIL=$((FAIL+1))
fi

# 3. Bootstrap server reachable?
echo "[3/5] Testing connectivity to bootstrap server..."
HOST=$(echo "$BOOTSTRAP" | cut -d: -f1)
PORT=$(echo "$BOOTSTRAP" | cut -d: -f2)
if nc -z -w 5 "$HOST" "$PORT" 2>/dev/null; then
    echo "  ✅ PASS — $BOOTSTRAP reachable"
    PASS=$((PASS+1))
else
    echo "  ❌ FAIL — Cannot reach $BOOTSTRAP"
    echo "  → Check firewall/VPN allows outbound port $PORT"
    FAIL=$((FAIL+1))
fi

# 4. API keys exist?
echo "[4/5] Checking API keys..."
KEY_COUNT=$(confluent api-key list --output json 2>/dev/null | jq 'length' 2>/dev/null)
if [ -n "$KEY_COUNT" ] && [ "$KEY_COUNT" -gt 0 ]; then
    echo "  ✅ PASS — $KEY_COUNT API key(s) found"
    PASS=$((PASS+1))
else
    echo "  ❌ FAIL — No API keys found"
    echo "  → confluent api-key create --resource <cluster-id>"
    FAIL=$((FAIL+1))
fi

# 5. Token file uploaded?
echo "[5/5] Checking token file..."
echo "  ⚠️  MANUAL CHECK — Verify that the .txt file with the API Secret"
echo "     was placed in the root of your file integration bucket."
echo "     Pismo's automation reads and deletes it within minutes."

# Save diagnostic output to log file
LOG_FILE="/tmp/pismo-diagnostic-$(date +%Y%m%d-%H%M%S).log"

# Capture everything to log file
{
    echo "=================================================================="
    echo "  PISMO DATA PLATFORM — DIAGNOSTIC LOG"
    echo "=================================================================="
    echo ""
    echo "Timestamp: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
    echo "Integration Type: Confluent Cloud Kafka"
    echo ""
    echo "=== CONFIGURATION ==="
    echo "Org ID: $ORG_ID"
    echo "Bootstrap Server: $BOOTSTRAP_SERVER"
    echo "Cluster ID: $CLUSTER_ID"
    echo "Topic: $TOPIC_NAME"
    echo "API Key: ${API_KEY:0:8}... (truncated for security)"
    echo ""
    echo "=== TEST RESULTS ==="
    echo "Passed: $PASS"
    echo "Failed: $FAIL"
    # Note: We don't capture secrets
    echo ""
    echo "=== SECURITY NOTE ==="
    echo "API Secret is NOT captured in this log for security reasons."
} > "$LOG_FILE" 2>&1

# Display results summary
echo ""
echo "============================================================"
echo "  RESULTS: $PASS passed, $FAIL failed"
echo "============================================================"

if [ $FAIL -eq 0 ]; then
    echo ""
    echo "  ✅ All checks passed — your infrastructure is correctly configured."
    echo ""
    echo "  If events are still not arriving, there may be a potential issue in:"
    echo "    • Kafka Producer component (API key authentication)"
    echo "    • Event Router configuration (org/destination mapping)"
    echo "    • Event generation (no events for your org in the selected period)"
    echo ""
else
    echo ""
    echo "  ❌ Some checks failed — review the errors above."
    echo ""
    echo "  These failures indicate configuration issues in your infrastructure."
    echo "  Follow the suggested fixes (→) for each failed check."
    echo "  After fixing, run this script again to verify."
    echo ""
fi

echo "============================================================"
echo "  📋 SUPPORT TICKET INSTRUCTIONS"
echo "============================================================"
echo ""
echo "  Diagnostic log saved to: $LOG_FILE"
echo ""
echo "  To open a support ticket:"
echo "  ┌─────────────────────────────────────────────────────────┐"
echo "  │  1. Portal: https://pismolabs.atlassian.net/servicedesk │"
echo "  │     /customer/portal/10                                 │"
echo "  │  2. Category: Settings → Data → Event Integration"
echo "  │  3. Attach the log file OR paste its content below      │"
echo "  │  ⚠️  DO NOT share your API Secret in the ticket"
echo "  └─────────────────────────────────────────────────────────┘"
echo ""
echo "  Copy to clipboard (macOS):  cat $LOG_FILE | pbcopy"
echo "  Copy to clipboard (Linux):  cat $LOG_FILE | xclip -selection clipboard"
echo ""
echo "  ────────────── LOG FILE CONTENT ──────────────"
echo ""
cat "$LOG_FILE"
echo ""
echo "  ──────────────────────────────────────────────"
echo ""```