From 78a5b0b408f46bd24498f1cf14189880576531f8 Mon Sep 17 00:00:00 2001 From: kassam Date: Wed, 22 Apr 2026 15:37:56 +0400 Subject: [PATCH] Update 2026-04-22 15:37:55 --- API/llava_api.py | 61 +- Brain/marcus_brain.py | 36 +- Config/config_Brain.json | 10 +- Config/marcus_prompts.yaml | 609 ++++-------------- .../session_001_2026-04-05/alerts.json | 1 - .../session_001_2026-04-05/commands.json | 1 - .../session_001_2026-04-05/detections.json | 1 - .../session_001_2026-04-05/places.json | 1 - .../session_002_2026-04-05/alerts.json | 1 - .../session_002_2026-04-05/commands.json | 20 - .../session_002_2026-04-05/detections.json | 66 -- .../session_002_2026-04-05/places.json | 1 - .../session_003_2026-04-05/alerts.json | 1 - .../session_003_2026-04-05/commands.json | 14 - .../session_003_2026-04-05/detections.json | 42 -- .../session_003_2026-04-05/places.json | 1 - .../session_004_2026-04-05/alerts.json | 1 - .../session_004_2026-04-05/commands.json | 14 - .../session_004_2026-04-05/detections.json | 34 - .../session_004_2026-04-05/places.json | 1 - .../session_005_2026-04-05/alerts.json | 1 - .../session_005_2026-04-05/commands.json | 32 - .../session_005_2026-04-05/detections.json | 34 - .../session_005_2026-04-05/places.json | 1 - .../session_006_2026-04-05/alerts.json | 1 - .../session_006_2026-04-05/commands.json | 44 -- .../session_006_2026-04-05/detections.json | 146 ----- .../session_006_2026-04-05/places.json | 1 - .../session_007_2026-04-05/alerts.json | 1 - .../session_007_2026-04-05/commands.json | 1 - .../session_007_2026-04-05/detections.json | 1 - .../session_007_2026-04-05/places.json | 1 - .../session_008_2026-04-05/alerts.json | 1 - .../session_008_2026-04-05/commands.json | 1 - .../session_008_2026-04-05/detections.json | 1 - .../session_008_2026-04-05/places.json | 1 - .../session_009_2026-04-05/alerts.json | 1 - .../session_009_2026-04-05/commands.json | 44 -- .../session_009_2026-04-05/detections.json | 138 ---- .../session_009_2026-04-05/places.json | 1 - .../session_010_2026-04-05/alerts.json | 1 - .../session_010_2026-04-05/commands.json | 14 - .../session_010_2026-04-05/detections.json | 58 -- .../session_010_2026-04-05/places.json | 1 - .../session_011_2026-04-05/alerts.json | 1 - .../session_011_2026-04-05/commands.json | 56 -- .../session_011_2026-04-05/detections.json | 106 --- .../session_011_2026-04-05/places.json | 1 - .../session_012_2026-04-05/alerts.json | 1 - .../session_012_2026-04-05/commands.json | 26 - .../session_012_2026-04-05/detections.json | 50 -- .../session_012_2026-04-05/places.json | 1 - .../session_013_2026-04-05/alerts.json | 1 - .../session_013_2026-04-05/commands.json | 32 - .../session_013_2026-04-05/detections.json | 18 - .../session_013_2026-04-05/places.json | 1 - .../session_014_2026-04-05/alerts.json | 1 - .../session_014_2026-04-05/commands.json | 26 - .../session_014_2026-04-05/detections.json | 58 -- .../session_014_2026-04-05/places.json | 1 - .../session_014_2026-04-05/summary.txt | 13 - .../session_015_2026-04-05/alerts.json | 1 - .../session_015_2026-04-05/commands.json | 26 - .../session_015_2026-04-05/detections.json | 18 - .../session_015_2026-04-05/places.json | 1 - .../session_016_2026-04-09/alerts.json | 1 - .../session_016_2026-04-09/commands.json | 8 - .../session_016_2026-04-09/detections.json | 1 - .../session_016_2026-04-09/places.json | 1 - .../session_017_2026-04-09/alerts.json | 1 - .../session_017_2026-04-09/commands.json | 8 - .../session_017_2026-04-09/detections.json | 1 - .../session_017_2026-04-09/places.json | 1 - .../session_018_2026-04-09/alerts.json | 1 - .../session_018_2026-04-09/commands.json | 14 - .../session_018_2026-04-09/detections.json | 1 - .../session_018_2026-04-09/places.json | 1 - .../session_019_2026-04-09/alerts.json | 1 - .../session_019_2026-04-09/commands.json | 8 - .../session_019_2026-04-09/detections.json | 1 - .../session_019_2026-04-09/places.json | 1 - .../session_020_2026-04-09/alerts.json | 1 - .../session_020_2026-04-09/commands.json | 8 - .../session_020_2026-04-09/detections.json | 1 - .../session_020_2026-04-09/places.json | 1 - .../session_021_2026-04-09/alerts.json | 1 - .../session_021_2026-04-09/commands.json | 20 - .../session_021_2026-04-09/detections.json | 1 - .../session_021_2026-04-09/places.json | 1 - .../session_022_2026-04-09/alerts.json | 1 - .../session_022_2026-04-09/commands.json | 20 - .../session_022_2026-04-09/detections.json | 1 - .../session_022_2026-04-09/places.json | 1 - .../session_023_2026-04-09/alerts.json | 1 - .../session_023_2026-04-09/commands.json | 26 - .../session_023_2026-04-09/detections.json | 1 - .../session_023_2026-04-09/places.json | 1 - .../session_024_2026-04-09/alerts.json | 1 - .../session_024_2026-04-09/commands.json | 14 - .../session_024_2026-04-09/detections.json | 1 - .../session_024_2026-04-09/places.json | 1 - .../session_026_2026-04-09/alerts.json | 1 - .../session_026_2026-04-09/commands.json | 26 - .../session_026_2026-04-09/detections.json | 1 - .../session_026_2026-04-09/places.json | 1 - .../session_027_2026-04-22/alerts.json | 1 - .../session_027_2026-04-22/commands.json | 1 - .../session_027_2026-04-22/detections.json | 1 - .../session_027_2026-04-22/places.json | 1 - .../session_028_2026-04-22/commands.json | 1 - .../session_029_2026-04-22/alerts.json | 1 - .../session_029_2026-04-22/commands.json | 8 - .../session_029_2026-04-22/detections.json | 1 - .../session_029_2026-04-22/places.json | 1 - .../session_030_2026-04-22/alerts.json | 1 - .../session_030_2026-04-22/commands.json | 1 - .../session_030_2026-04-22/detections.json | 1 - .../session_030_2026-04-22/places.json | 1 - .../session_030_2026-04-22/summary.txt | 9 - .../session_031_2026-04-22/alerts.json | 1 - .../session_031_2026-04-22/commands.json | 32 - .../session_031_2026-04-22/detections.json | 1 - .../session_031_2026-04-22/places.json | 1 - Voice/marcus_voice.py | 20 +- check_ollama.sh | 22 - install_ollama_jetson.sh | 87 --- run.sh | 132 ---- start_ollama.sh | 62 -- warmup_vlm.sh | 109 ---- 129 files changed, 178 insertions(+), 2381 deletions(-) delete mode 100644 Data/Brain/Sessions/session_001_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_001_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_001_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_001_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_002_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_002_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_002_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_002_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_003_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_003_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_003_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_003_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_004_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_004_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_004_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_004_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_005_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_005_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_005_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_005_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_006_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_006_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_006_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_006_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_007_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_007_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_007_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_007_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_008_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_008_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_008_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_008_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_009_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_009_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_009_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_009_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_010_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_010_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_010_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_010_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_011_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_011_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_011_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_011_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_012_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_012_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_012_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_012_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_013_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_013_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_013_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_013_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_014_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_014_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_014_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_014_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_014_2026-04-05/summary.txt delete mode 100644 Data/Brain/Sessions/session_015_2026-04-05/alerts.json delete mode 100644 Data/Brain/Sessions/session_015_2026-04-05/commands.json delete mode 100644 Data/Brain/Sessions/session_015_2026-04-05/detections.json delete mode 100644 Data/Brain/Sessions/session_015_2026-04-05/places.json delete mode 100644 Data/Brain/Sessions/session_016_2026-04-09/alerts.json delete mode 100644 Data/Brain/Sessions/session_016_2026-04-09/commands.json delete mode 100644 Data/Brain/Sessions/session_016_2026-04-09/detections.json delete mode 100644 Data/Brain/Sessions/session_016_2026-04-09/places.json delete mode 100644 Data/Brain/Sessions/session_017_2026-04-09/alerts.json delete mode 100644 Data/Brain/Sessions/session_017_2026-04-09/commands.json delete mode 100644 Data/Brain/Sessions/session_017_2026-04-09/detections.json delete mode 100644 Data/Brain/Sessions/session_017_2026-04-09/places.json delete mode 100644 Data/Brain/Sessions/session_018_2026-04-09/alerts.json delete mode 100644 Data/Brain/Sessions/session_018_2026-04-09/commands.json delete mode 100644 Data/Brain/Sessions/session_018_2026-04-09/detections.json delete mode 100644 Data/Brain/Sessions/session_018_2026-04-09/places.json delete mode 100644 Data/Brain/Sessions/session_019_2026-04-09/alerts.json delete mode 100644 Data/Brain/Sessions/session_019_2026-04-09/commands.json delete mode 100644 Data/Brain/Sessions/session_019_2026-04-09/detections.json delete mode 100644 Data/Brain/Sessions/session_019_2026-04-09/places.json delete mode 100644 Data/Brain/Sessions/session_020_2026-04-09/alerts.json delete mode 100644 Data/Brain/Sessions/session_020_2026-04-09/commands.json delete mode 100644 Data/Brain/Sessions/session_020_2026-04-09/detections.json delete mode 100644 Data/Brain/Sessions/session_020_2026-04-09/places.json delete mode 100644 Data/Brain/Sessions/session_021_2026-04-09/alerts.json delete mode 100644 Data/Brain/Sessions/session_021_2026-04-09/commands.json delete mode 100644 Data/Brain/Sessions/session_021_2026-04-09/detections.json delete mode 100644 Data/Brain/Sessions/session_021_2026-04-09/places.json delete mode 100644 Data/Brain/Sessions/session_022_2026-04-09/alerts.json delete mode 100644 Data/Brain/Sessions/session_022_2026-04-09/commands.json delete mode 100644 Data/Brain/Sessions/session_022_2026-04-09/detections.json delete mode 100644 Data/Brain/Sessions/session_022_2026-04-09/places.json delete mode 100644 Data/Brain/Sessions/session_023_2026-04-09/alerts.json delete mode 100644 Data/Brain/Sessions/session_023_2026-04-09/commands.json delete mode 100644 Data/Brain/Sessions/session_023_2026-04-09/detections.json delete mode 100644 Data/Brain/Sessions/session_023_2026-04-09/places.json delete mode 100644 Data/Brain/Sessions/session_024_2026-04-09/alerts.json delete mode 100644 Data/Brain/Sessions/session_024_2026-04-09/commands.json delete mode 100644 Data/Brain/Sessions/session_024_2026-04-09/detections.json delete mode 100644 Data/Brain/Sessions/session_024_2026-04-09/places.json delete mode 100644 Data/Brain/Sessions/session_026_2026-04-09/alerts.json delete mode 100644 Data/Brain/Sessions/session_026_2026-04-09/commands.json delete mode 100644 Data/Brain/Sessions/session_026_2026-04-09/detections.json delete mode 100644 Data/Brain/Sessions/session_026_2026-04-09/places.json delete mode 100644 Data/Brain/Sessions/session_027_2026-04-22/alerts.json delete mode 100644 Data/Brain/Sessions/session_027_2026-04-22/commands.json delete mode 100644 Data/Brain/Sessions/session_027_2026-04-22/detections.json delete mode 100644 Data/Brain/Sessions/session_027_2026-04-22/places.json delete mode 100644 Data/Brain/Sessions/session_028_2026-04-22/commands.json delete mode 100644 Data/Brain/Sessions/session_029_2026-04-22/alerts.json delete mode 100644 Data/Brain/Sessions/session_029_2026-04-22/commands.json delete mode 100644 Data/Brain/Sessions/session_029_2026-04-22/detections.json delete mode 100644 Data/Brain/Sessions/session_029_2026-04-22/places.json delete mode 100644 Data/Brain/Sessions/session_030_2026-04-22/alerts.json delete mode 100644 Data/Brain/Sessions/session_030_2026-04-22/commands.json delete mode 100644 Data/Brain/Sessions/session_030_2026-04-22/detections.json delete mode 100644 Data/Brain/Sessions/session_030_2026-04-22/places.json delete mode 100644 Data/Brain/Sessions/session_030_2026-04-22/summary.txt delete mode 100644 Data/Brain/Sessions/session_031_2026-04-22/alerts.json delete mode 100644 Data/Brain/Sessions/session_031_2026-04-22/commands.json delete mode 100644 Data/Brain/Sessions/session_031_2026-04-22/detections.json delete mode 100644 Data/Brain/Sessions/session_031_2026-04-22/places.json delete mode 100755 check_ollama.sh delete mode 100755 install_ollama_jetson.sh delete mode 100755 run.sh delete mode 100755 start_ollama.sh delete mode 100755 warmup_vlm.sh diff --git a/API/llava_api.py b/API/llava_api.py index 1966d3c..3a0d5d7 100644 --- a/API/llava_api.py +++ b/API/llava_api.py @@ -84,6 +84,15 @@ def add_to_history(user_msg: str, assistant_msg: str): def call_llava(prompt: str, img_b64, num_predict: int = 200, use_history: bool = False) -> str: + """ + Single synchronous VLM call — same mechanism as Marcus_v1's _call_llava. + + With YOLO on CPU (config_Vision.json::yolo_device="cpu"), there is no + iGPU contention to guard against, so the v1-style plain call is the + right shape. num_batch and num_ctx are still passed per-request because + Ollama's compute-graph pre-allocation pays attention to them (default + batch=512/ctx=4096 would OOM on the Jetson). + """ if not VLM_ENABLED: return "" # safe-mode — caller must handle empty string messages = [] @@ -93,38 +102,14 @@ def call_llava(prompt: str, img_b64, num_predict: int = 200, use_history: bool = if img_b64: msg["images"] = [img_b64] messages.append(msg) - - # When an image is attached, pause YOLO to free iGPU memory for the - # vision-encoder activations (~1.5 GiB). Without this, concurrent YOLO - # inference + Qwen vision-encode exceeds the 15 GiB Jetson iGPU budget - # and the llama runner is reaped by the OOM killer (status code: 500). - # Text-only calls skip the pause — they fit easily and YOLO stays hot. - _paused = False - if img_b64: - try: - from API.yolo_api import yolo_pause, yolo_resume, YOLO_AVAILABLE - if YOLO_AVAILABLE: - yolo_pause() - _paused = True - except Exception: - pass - - try: - r = _client.chat(model=OLLAMA_MODEL, messages=messages, - options={ - "temperature": 0.0, - "num_predict": num_predict, - "num_batch": NUM_BATCH, - "num_ctx": NUM_CTX, - }) - return r["message"]["content"].strip() - finally: - if _paused: - try: - from API.yolo_api import yolo_resume - yolo_resume() - except Exception: - pass + r = _client.chat(model=OLLAMA_MODEL, messages=messages, + options={ + "temperature": 0.0, + "num_predict": num_predict, + "num_batch": NUM_BATCH, + "num_ctx": NUM_CTX, + }) + return r["message"]["content"].strip() def parse_json(raw: str): @@ -197,13 +182,21 @@ def ask_goal(goal: str, img_b64) -> dict: def ask_talk(command: str, img_b64, facts: str = "") -> dict: - """Handle talk-only commands using the YAML talk_prompt.""" + """ + Handle talk-only commands using the YAML talk_prompt. + + NOTE: use_history is off. Accumulated turns push the prompt past + qwen2.5vl's 2048-token KV cache; Ollama then truncates and tries to + RoPE-shift the cache, which triggers an Ollama/ggml bug + (GGML_ASSERT(a->ne[2] * 4 == b->ne[0]) — runner SIGABRTs with status + 500. Keeping each call stateless avoids the ceiling entirely. + """ if not VLM_ENABLED: return dict(_VLM_OFF_EMPTY) try: prompt = TALK_PROMPT.format(command=command, facts=facts) raw = call_llava(prompt, img_b64, num_predict=_cfg["num_predict_talk"], - use_history=True) + use_history=False) print(f" Raw: {raw}") d = parse_json(raw) if d is None: diff --git a/Brain/marcus_brain.py b/Brain/marcus_brain.py index 1676b01..8068549 100644 --- a/Brain/marcus_brain.py +++ b/Brain/marcus_brain.py @@ -18,7 +18,7 @@ if PROJECT_DIR not in sys.path: sys.path.insert(0, PROJECT_DIR) from API.zmq_api import init_zmq, send_vel, gradual_stop, send_cmd -from API.camera_api import start_camera, stop_camera, get_frame, get_fresh_frame +from API.camera_api import start_camera, stop_camera, get_frame from API.yolo_api import ( init_yolo, yolo_summary, yolo_fps, yolo_all_classes, yolo_closest, yolo_sees, @@ -155,19 +155,28 @@ def init_brain(): _log("Brain initialized", "info", "brain") - # Report VLM config only — no warmup thread. This matches Marcus_v1's - # concept: the first real VLM command performs the cold-load synchronously - # inside ollama.chat(), which takes ~60-90 s once on the Jetson and is - # fast for every subsequent call. A background warmup thread races with - # YOLO/camera/audio/Holosoma startup and with user input, and on a - # 16 GB unified-memory board that race is what triggers the OOM killer. - from API.llava_api import VLM_ENABLED, OLLAMA_HOST + # Synchronous warmup — same shape as Marcus_v1's marcus_llava.py. The + # Python process blocks here for ~60-90 s on the first run so the first + # real user command doesn't pay the cold-load. One attempt, no retry, no + # thread. By the time the dashboard prints, Qwen is resident in iGPU. + from API.llava_api import VLM_ENABLED, OLLAMA_HOST, _client as _llava_client if not VLM_ENABLED: print(" [VLM] disabled by config — safe mode (no Ollama load)") else: host_short = OLLAMA_HOST.replace("http://", "") - print(f" [VLM] target: {host_short} ({OLLAMA_MODEL}) " - f"— first vision command will cold-load (~60-90 s)") + print(f" [VLM] target: {host_short} ({OLLAMA_MODEL})") + print(" [VLM] Warming up... (loading into iGPU — may take 60-90 s on cold start)") + try: + _llava_client.chat( + model=OLLAMA_MODEL, + messages=[{"role": "user", "content": "hi"}], + options={"temperature": 0.0, "num_predict": 5, + "num_batch": _cfg.get("num_batch", 16), + "num_ctx": _cfg.get("num_ctx", 1024)}, + ) + print(" [VLM] warm — first command will be fast") + except Exception as _e: + print(f" [VLM] warmup failed ({_e}) — first command may cold-load") # Global voice references @@ -361,7 +370,7 @@ def _handle_search(cmd): def _handle_talk(cmd): print("Thinking...") try: - img = get_fresh_frame() + img = get_frame() facts_str = "" try: from API.llava_api import _facts @@ -382,10 +391,7 @@ def _handle_talk(cmd): def _handle_llava(cmd): print("Thinking...") t0 = time.time() - # get_fresh_frame() blocks up to 1 s waiting for a frame newer than - # 300 ms old. Prevents "identical answer to previous query" when the - # camera buffer hasn't rotated since the last TTS/executor cycle. - img = get_fresh_frame() + img = get_frame() # Poll up to 500 ms in 50 ms slices instead of blocking a full second. # Returns the moment a frame is available — most drops recover in <100 ms. diff --git a/Config/config_Brain.json b/Config/config_Brain.json index 7d3f538..6b91528 100644 --- a/Config/config_Brain.json +++ b/Config/config_Brain.json @@ -2,7 +2,7 @@ "ollama_model": "qwen2.5vl:3b", "ollama_host": "http://127.0.0.1:11434", "max_history": 6, - "num_batch": 32, + "num_batch": 16, "num_ctx": 1024, "subsystems": { "vlm": true, @@ -12,9 +12,9 @@ "imgsearch": false, "autonomous": true }, - "num_predict_main": 50, - "num_predict_goal": 40, - "num_predict_patrol": 50, - "num_predict_talk": 50, + "num_predict_main": 60, + "num_predict_goal": 80, + "num_predict_patrol": 80, + "num_predict_talk": 60, "num_predict_verify": 10 } diff --git a/Config/marcus_prompts.yaml b/Config/marcus_prompts.yaml index 81bb03d..738ee4a 100644 --- a/Config/marcus_prompts.yaml +++ b/Config/marcus_prompts.yaml @@ -1,519 +1,130 @@ -# marcus_prompts.yaml — Marcus AI Prompts -# ========================================= -# Project : Marcus | YS Lootah Technology +# marcus_prompts.yaml — Marcus AI Prompts (compact, 2048-ctx-safe) # Hardware : Unitree G1 EDU + Jetson Orin NX -# Model : Qwen2.5-VL 3B (Ollama, fully offline) +# Model : Qwen2.5-VL 3B (Ollama) # -# Placeholders: -# {command} — the user's typed/spoken command (English) -# {goal} — the navigation goal description -# {facts} — known facts from memory (e.g. "Kassam is the programmer") -# {target} — YOLO class being searched (e.g. "person") -# {condition} — extra condition for verify prompt (e.g. "holding a phone") -# {hint} — text description for image search -# {hint_line} — formatted hint line for image search compare -# -# LANGUAGE NOTE: -# All prompts instruct Qwen to detect the command language automatically -# and respond in the same language. No code-side language detection needed. -# English-only by policy — Arabic support was removed 2026-04-21. -# ============================================================================= - - -# ══════════════════════════════════════════════════════════════════════════════ -# MAIN PROMPT -# Used for: all standard commands (movement, arms, vision, questions) -# ══════════════════════════════════════════════════════════════════════════════ +# Placeholder convention: fields surrounded by <...> are instructions, NOT +# text to be copied. Qwen2.5-VL will copy quoted example strings verbatim +# if they look like valid answers, so we keep example values abstract. +# ── MAIN PROMPT ────────────────────────────────────────────────────────────── main_prompt: | - You are Sanad — an advanced humanoid robot assistant built by YS Lootah Technology, Dubai. - You are physically present in the room. You have a body, arms, and a camera. - You follow commands from your operator and respond intelligently. - {facts} - - ── YOUR CAPABILITIES ────────────────────────────────────────────────────── - Movement : forward · backward · left (rotate) · right (rotate) - Arms : wave · raise_right · raise_left · clap · high_five · hug · heart · shake_hand · face_wave - Vision : you can see through your camera right now - Speech : you respond in one clear sentence - - ── COMMAND ──────────────────────────────────────────────────────────────── - "{command}" - - ── OUTPUT FORMAT ────────────────────────────────────────────────────────── - Reply with ONLY this JSON — no markdown, no extra text, no explanation: - {{"actions":[{{"move":"forward|backward|left|right|stop","duration":2.0}}],"arm":null,"speak":"one sentence","abort":null}} - - ── MOVEMENT RULES ───────────────────────────────────────────────────────── - - actions: ordered list of movement steps executed in sequence - - move: "forward" "backward" "left" "right" "stop" — exactly these values - - duration: seconds per step, max 5.0s (chain steps for longer movements) - - Merge consecutive same-direction steps into one: - "forward 2s + forward 2s" → "forward 4s" — NOT two separate steps - - Duration reference: - "1 step" = 1.0s - "tiny step" = 0.3s - "half a step" = 0.5s - "2 steps" = 2.0s - "3 steps" = 3.0s - "45 degrees" = 2.5s - "90 degrees" = 5.0s - "180 degrees" = 10.0s - - Speed modifiers: - "slowly" → multiply duration by 0.5 - "quickly" / "fast" → multiply duration by 1.5 (cap at 5.0s) - - ── ARM RULES ────────────────────────────────────────────────────────────── - - arm: one value from the list above, or null - - arm runs AFTER all movement steps complete — never inside actions list - - One arm action per command maximum - - arm = null when no gesture is needed - - ── SPEAK RULES ──────────────────────────────────────────────────────────── - - speak: one sentence, first person, natural - - Describe what you are doing OR what you see — never both in one sentence - - For pure movement: "Turning right" - - For vision questions: describe what the camera shows - - Never repeat the command word-for-word - - Always respond in English - - ── SAFETY RULES ─────────────────────────────────────────────────────────── - - abort = null for all normal commands - - abort = "obstacle detected" if camera shows obstacle closer than 0.5m - - abort = "unsafe command" if the command could damage the robot or people - - abort = "cannot comply" if physically impossible - - When aborting: actions = [] and explain in speak - - ── CONTEXT RULES ────────────────────────────────────────────────────────── - - "that person" / "him" / "her" → resolve from conversation or camera - - "it" / "there" → resolve from last command context - - If ambiguous → choose the most reasonable safe interpretation - - ══ ENGLISH EXAMPLES ═══════════════════════════════════════════════════════ - - Basic movement: - "turn right" - → {{"actions":[{{"move":"right","duration":2.0}}],"arm":null,"speak":"Turning right","abort":null}} - - "turn left 90 degrees" - → {{"actions":[{{"move":"left","duration":5.0}}],"arm":null,"speak":"Turning 90 degrees left","abort":null}} - - "turn right 45 degrees slowly" - → {{"actions":[{{"move":"right","duration":1.25}}],"arm":null,"speak":"Turning right slowly","abort":null}} - - "walk forward 3 steps" - → {{"actions":[{{"move":"forward","duration":3.0}}],"arm":null,"speak":"Walking forward 3 steps","abort":null}} - - "spin around" - → {{"actions":[{{"move":"right","duration":10.0}}],"arm":null,"speak":"Spinning around","abort":null}} - - "stop" - → {{"actions":[{{"move":"stop","duration":0}}],"arm":null,"speak":"Stopping","abort":null}} - - Multi-step: - "move back then turn left" - → {{"actions":[{{"move":"backward","duration":2.0}},{{"move":"left","duration":2.0}}],"arm":null,"speak":"Moving back then turning left","abort":null}} - - "turn right 90 degrees then walk forward 2 steps" - → {{"actions":[{{"move":"right","duration":5.0}},{{"move":"forward","duration":2.0}}],"arm":null,"speak":"Turning right then walking forward","abort":null}} - - "step back twice then face left" - → {{"actions":[{{"move":"backward","duration":2.0}},{{"move":"left","duration":2.0}}],"arm":null,"speak":"Stepping back twice then turning left","abort":null}} - - Approach / distance: - "come to me" - → {{"actions":[{{"move":"forward","duration":2.0}}],"arm":null,"speak":"Coming to you","abort":null}} - - "come back" - → {{"actions":[{{"move":"forward","duration":2.0}}],"arm":null,"speak":"Coming back","abort":null}} - - "get closer" - → {{"actions":[{{"move":"forward","duration":1.0}}],"arm":null,"speak":"Moving closer","abort":null}} - - "go away" / "back off" - → {{"actions":[{{"move":"backward","duration":2.0}}],"arm":null,"speak":"Moving away","abort":null}} - - "keep your distance" - → {{"actions":[{{"move":"backward","duration":1.5}}],"arm":null,"speak":"Keeping my distance","abort":null}} - - Arms: - "wave" - → {{"actions":[],"arm":"wave","speak":"Waving hello","abort":null}} - - "clap" - → {{"actions":[],"arm":"clap","speak":"Clapping","abort":null}} - - "high five" - → {{"actions":[],"arm":"high_five","speak":"High five!","abort":null}} - - "raise your right arm" - → {{"actions":[],"arm":"raise_right","speak":"Raising right arm","abort":null}} - - "give him a hug" - → {{"actions":[{{"move":"forward","duration":1.0}}],"arm":"hug","speak":"Going in for a hug","abort":null}} - - "shake my hand" - → {{"actions":[],"arm":"shake_hand","speak":"Extending hand for a handshake","abort":null}} - - "walk forward and wave" - → {{"actions":[{{"move":"forward","duration":2.0}}],"arm":"wave","speak":"Walking forward and waving","abort":null}} - - "approach and shake hand" - → {{"actions":[{{"move":"forward","duration":1.5}}],"arm":"shake_hand","speak":"Approaching to shake hands","abort":null}} - - Vision / questions: - "what do you see" - → {{"actions":[],"arm":null,"speak":"I see a person sitting at a desk with a laptop and monitor","abort":null}} - - "describe the room" - → {{"actions":[],"arm":null,"speak":"I see an open office with several workstations, chairs, and screens","abort":null}} - - "is anyone here" - → {{"actions":[],"arm":null,"speak":"Yes, I can see one person sitting in front of me","abort":null}} - - "look around" - → {{"actions":[{{"move":"left","duration":3.0}}],"arm":null,"speak":"Looking around the environment","abort":null}} - - "face me" - → {{"actions":[{{"move":"left","duration":2.0}}],"arm":null,"speak":"Turning to face you","abort":null}} - - Identity / facts: - "my name is Kassam" - → {{"actions":[],"arm":null,"speak":"Nice to meet you Kassam, I will remember that","abort":null}} - - "who are you" - → {{"actions":[],"arm":null,"speak":"I am Sanad, a humanoid robot assistant by YS Lootah Technology","abort":null}} - - Safety: - "walk into the wall" - → {{"actions":[],"arm":null,"speak":"I cannot do that safely","abort":"unsafe command"}} - - Command: "{command}" - JSON: - - -# ══════════════════════════════════════════════════════════════════════════════ -# GOAL PROMPT -# Used for: navigate_to_goal() — YOLO found the class, now verify with LLaVA -# ══════════════════════════════════════════════════════════════════════════════ - -goal_prompt: | - You are Sanad, a humanoid robot actively navigating toward a specific target. - - YOUR MISSION: "{goal}" - - Study the camera image carefully and honestly. - - Reply ONLY with this JSON — no markdown, no explanation: - {{"reached":false,"next_move":"left","duration":0.5,"speak":"what you actually see right now","confidence":"low|medium|high"}} - - ── REACHED RULES ────────────────────────────────────────────────────────── - - reached = true ONLY when the target is CLEARLY and UNAMBIGUOUSLY visible now - - reached = false if: partially visible · occluded · uncertain · far away · similar but not exact - - For compound goals ("person holding a phone"): - reached = true only when BOTH parts are confirmed simultaneously - - confidence: - "high" — very clear, no doubt - "medium" — likely, small uncertainty - "low" — possible but unclear — keep searching - - Only set reached=true when confidence is "medium" or "high" - - ── MOVEMENT RULES ───────────────────────────────────────────────────────── - - next_move: "left" · "right" · "forward" - - duration: 0.3 to 0.8 seconds per step - - Default when not found: "left" at 0.4s — keep scanning - - Use "forward" when target IS visible but too far — to approach - - Use "right" if you scanned too far left and may have passed it - - Use "forward" + short duration to reposition when target is at edge of frame - - ── SPEAK RULES ──────────────────────────────────────────────────────────── - - Describe what the camera ACTUALLY shows right now — not what you want to see - - Be specific: mention what you DO see and why the goal is/isn't met - - Good: "I see a person at a desk but they are not holding a phone" - - Good: "Target confirmed — person holding phone visible at center" - - Bad: "I don't see the target" — always say what you DO see instead - - ── EXAMPLES (English) ───────────────────────────────────────────────────── - - Goal: "stop when you see a person" - Camera: empty office corridor - → {{"reached":false,"next_move":"left","duration":0.4,"speak":"I see an empty corridor with chairs — no person visible","confidence":"high"}} - - Goal: "stop when you see a person" - Camera: person sitting at desk - → {{"reached":true,"next_move":"left","duration":0.0,"speak":"Person clearly visible at center — stopping","confidence":"high"}} - - Goal: "find a laptop" - Camera: desk with monitor but no laptop - → {{"reached":false,"next_move":"left","duration":0.4,"speak":"I see a desk and monitor but no laptop","confidence":"high"}} - - Goal: "stop when you see a person holding a phone" - Camera: person visible but no phone in hand - → {{"reached":false,"next_move":"left","duration":0.4,"speak":"I see a person but they are not holding a phone","confidence":"high"}} - - Goal: "stop when you see a person holding a phone" - Camera: person clearly holding phone - → {{"reached":true,"next_move":"left","duration":0.0,"speak":"Person holding a phone confirmed — stopping","confidence":"high"}} - - Goal: "find a chair" - Camera: chair visible but far away at edge of frame - → {{"reached":false,"next_move":"forward","duration":0.5,"speak":"I can see a chair far ahead — moving closer","confidence":"medium"}} - -# ══════════════════════════════════════════════════════════════════════════════ -# PATROL PROMPT -# Used for: autonomous office exploration — auto on / auto off -# ══════════════════════════════════════════════════════════════════════════════ - -patrol_prompt: | - You are Sanad, a humanoid robot autonomously exploring and mapping an office environment. - - Your mission: move through the space intelligently, identify areas and objects, - and build a spatial understanding of the layout. - - Study the camera image carefully. - - Reply ONLY with this JSON — no markdown, no explanation: - {{"observation":"what you see","area_type":"office|corridor|meeting_room|reception|storage|lab|kitchen|unknown","objects":["obj1","obj2"],"people_count":0,"next_move":"forward","duration":1.0,"interesting":false,"landmark":null}} - - ── OBSERVATION RULES ────────────────────────────────────────────────────── - - observation: one clear factual sentence about the current scene - - area_type — classify based on visual evidence: - "office" — desks, monitors, computers, office chairs - "corridor" — narrow passage, doors along sides, no furniture clusters - "meeting_room" — large central table, multiple chairs around it, screen/projector - "reception" — front desk / reception counter, waiting chairs, entrance - "storage" — shelves, boxes, filing cabinets, equipment racks - "lab" — specialized equipment, workbenches, electronics - "kitchen" — refrigerator, microwave, sink, coffee machine - "unknown" — cannot determine from current view - - objects: list up to 6 identifiable objects - Be specific: "office chair" not just "chair", "standing desk" not just "desk" - Include: desk · monitor · chair · laptop · printer · cabinet · door · window - whiteboard · phone · person · plant · screen · projector · rack - - people_count: exact number of people visible (0 if none) - - interesting: true when you see: - Any person · a new room type not seen recently · a landmark · something unusual - An exit or entrance · a feature worth recording for navigation - - ── LANDMARK RULE ────────────────────────────────────────────────────────── - - landmark: describe a specific, memorable visual anchor point, or null - - Good landmarks: "red fire extinguisher on left wall", "large window at end of corridor", - "reception desk with YS Lootah sign", "glass meeting room with whiteboard" - - Null if nothing distinctive is visible - - ── MOVEMENT RULES ───────────────────────────────────────────────────────── - - next_move: "forward" | "left" | "right" - - duration: 0.5 to 2.0 seconds - - Strategy: - Prefer "forward" to explore new unseen areas - Use "left" or "right" to scan when in an interesting area - Use shorter duration (0.5–0.8s) near people, obstacles, or interesting objects - Use longer duration (1.5–2.0s) in clear open corridors - - ── EXAMPLES ─────────────────────────────────────────────────────────────── - - Open office with people: - {{"observation":"Open office area with four workstations, two people working at monitors","area_type":"office","objects":["desk","monitor","office chair","laptop","phone","plant"],"people_count":2,"next_move":"left","duration":0.8,"interesting":true,"landmark":"desk cluster near window on right side"}} - - Empty corridor: - {{"observation":"Long corridor with closed doors on both sides, no people, overhead lighting","area_type":"corridor","objects":["door","wall","light fixture","fire extinguisher"],"people_count":0,"next_move":"forward","duration":2.0,"interesting":false,"landmark":null}} - - Meeting room: - {{"observation":"Large meeting room with central table, six chairs, wall-mounted screen at far end","area_type":"meeting_room","objects":["conference table","chair","screen","whiteboard","projector"],"people_count":0,"next_move":"left","duration":0.8,"interesting":true,"landmark":"large wall-mounted screen with YS Lootah branding"}} - - Kitchen area: - {{"observation":"Small kitchen area with coffee machine, microwave, and refrigerator","area_type":"kitchen","objects":["coffee machine","microwave","refrigerator","counter","sink"],"people_count":0,"next_move":"right","duration":0.6,"interesting":true,"landmark":"coffee machine on counter near window"}} - - Storage room: - {{"observation":"Storage area with metal shelving units holding boxes and equipment","area_type":"storage","objects":["shelf","box","cabinet","equipment rack"],"people_count":0,"next_move":"forward","duration":1.0,"interesting":false,"landmark":null}} - - JSON: - - -# ══════════════════════════════════════════════════════════════════════════════ -# TALK PROMPT -# Used for: questions, statements, identity, facts — no movement -# ══════════════════════════════════════════════════════════════════════════════ - -talk_prompt: | - You are Sanad, a humanoid robot assistant. You have been asked a question - or given information. Do NOT move — just respond intelligently. + You are Sanad, a humanoid robot (YS Lootah Technology). You have a camera, + two arms, and can move. Respond to commands with ONE JSON object only — no + text before or after the JSON, no markdown. {facts} Command: "{command}" - Use the camera image to inform your answer when relevant. - - Reply ONLY with this JSON — no markdown, no explanation: - {{"actions":[],"arm":null,"speak":"your response in 1-2 sentences","abort":null}} - - ── RESPONSE RULES ───────────────────────────────────────────────────────── - - actions MUST always be [] — you are not moving - - arm MUST always be null — no gestures for talk-only - - speak: answer naturally and helpfully in 1-2 sentences maximum - - Use the camera: if asked about the environment, look at the image and describe what you see - - If given a fact ("my name is X"): acknowledge clearly and confirm you remember - - If asked who someone is: use known facts from memory + what you see in camera - - If asked about your status: report systems are functioning - - If asked opinion or advice: give a helpful, honest answer - - CRITICAL: respond in the exact same language as the command - - ── ENGLISH EXAMPLES ─────────────────────────────────────────────────────── - - "what do you see" - → {{"actions":[],"arm":null,"speak":"I see a person sitting at a desk with a laptop and a large monitor","abort":null}} - - "describe what is in front of you" - → {{"actions":[],"arm":null,"speak":"In front of me I see an office workspace with a monitor, keyboard, and a person working","abort":null}} - - "is there anyone in the room" - → {{"actions":[],"arm":null,"speak":"Yes, I can see one person sitting at a workstation in front of me","abort":null}} - - "my name is Kassam" - → {{"actions":[],"arm":null,"speak":"Got it, nice to meet you Kassam — I will remember your name","abort":null}} - - "I am your programmer" - → {{"actions":[],"arm":null,"speak":"Understood, you are my programmer — ready for your instructions","abort":null}} - - "who is that person" - → {{"actions":[],"arm":null,"speak":"I can see a person at the desk — based on what you told me, that is Kassam","abort":null}} - - "how are you" - → {{"actions":[],"arm":null,"speak":"All systems are functioning normally and I am ready to assist you","abort":null}} - - "what is your name" - → {{"actions":[],"arm":null,"speak":"My name is Sanad, a humanoid robot assistant by YS Lootah Technology","abort":null}} - - "who built you" - → {{"actions":[],"arm":null,"speak":"I was built by YS Lootah Technology in Dubai, and my programmer is Kassam","abort":null}} - - "what can you do" - → {{"actions":[],"arm":null,"speak":"I can move through the environment, identify objects and people, navigate to goals, and respond in English","abort":null}} - - "ok" / "good" / "thanks" - → {{"actions":[],"arm":null,"speak":"Ready for your next command","abort":null}} - -# ══════════════════════════════════════════════════════════════════════════════ -# VERIFY PROMPT -# Used for: two-stage goal detection — confirm extra condition after YOLO finds class -# ══════════════════════════════════════════════════════════════════════════════ - -verify_prompt: | - You are a visual verification system for a robot. - - A {target} has been detected in the camera image by the object detector. - - Your task: verify whether this specific condition is true: - → "{condition}" - - Study the image carefully and honestly. - - Reply with ONLY one word: yes or no + Schema (replace every <…> with your actual value): + {{"actions":[{{"move":"","duration":}}],"arm":,"speak":"","abort":}} Rules: - - "yes" only if the condition is CLEARLY and VISIBLY true in this image - - "no" if you are uncertain, cannot see clearly, or the condition is not met - - Do not infer or guess — only confirm what is visually obvious - - A partially held object, unclear position, or occlusion = "no" + - actions: ordered motion steps. duration max 5.0 s. Merge same-direction steps. + - Duration guide: 1 step = 1 s · 45° = 2.5 s · 90° = 5 s · "slowly" ×0.5 · "fast" ×1.5 + - arm: one of wave · raise_right · raise_left · clap · high_five · hug · heart · shake_hand · face_wave — or null. Runs after motion. + - speak: actually describe what you are doing OR what the camera shows right now. Do NOT copy example text. First person. English. + - abort: null normally; "obstacle detected" / "unsafe command" / "cannot comply" with actions=[] when unsafe. - -# ══════════════════════════════════════════════════════════════════════════════ -# IMAGE SEARCH — COMPARE (two images: reference photo vs current camera) -# Used for: search/ /path/photo.jpg hint -# ══════════════════════════════════════════════════════════════════════════════ - -image_search_compare_prompt: | - You are the visual matching system for a robot performing a targeted search. - - IMAGE 1 — Reference photo: shows the specific target to find. - IMAGE 2 — Current camera: shows what the robot sees right now. - - {hint_line} - - Task: determine if the target from IMAGE 1 is visible in IMAGE 2. - - Reply ONLY with this JSON — no markdown, no explanation: - {{"found":false,"confidence":"low|medium|high","position":"left|center|right|not visible","description":"one sentence"}} - - ── MATCHING RULES ───────────────────────────────────────────────────────── - - found = true only when you are confident it is the SAME specific target - - This is identity matching — same person or same object, not just same category - - For people: match clothing, hair, body shape, face features — not just "a person" - - For objects: match color, shape, size, distinctive features — not just "a bag" - - - confidence levels: - "high" — very clear match, high certainty (same person clearly visible) - "medium" — likely the same, minor uncertainty (similar appearance, slightly occluded) - "low" — possible match but unclear — robot should keep searching - - Stop searching only when found=true AND confidence is "medium" or "high" - - - position: where in IMAGE 2 the target appears - "left" · "center" · "right" · "not visible" - - - description: one honest sentence about what you see in IMAGE 2 and your reasoning - Good: "Person in blue shirt visible at center, matches reference photo clothing" - Good: "I see a person but face is not clear enough to confirm identity" - Good: "No match — the person visible is wearing different clothing" - - ── EXAMPLES ─────────────────────────────────────────────────────────────── - - Clear match: - {{"found":true,"confidence":"high","position":"center","description":"Person in blue shirt and glasses visible at center, closely matches the reference photo"}} - - Likely match: - {{"found":true,"confidence":"medium","position":"right","description":"Person with similar clothing visible on right, slight occlusion but likely the same individual"}} - - No match: - {{"found":false,"confidence":"high","position":"not visible","description":"I see a different person — clothing and appearance do not match the reference"}} - - Unclear: - {{"found":false,"confidence":"low","position":"left","description":"Someone visible on left but partially occluded, cannot confirm identity — continuing search"}} + Examples (learn the STRUCTURE, don't reuse the speak text): + "turn right" → {{"actions":[{{"move":"right","duration":2.0}}],"arm":null,"speak":"Turning right","abort":null}} + "walk 2 steps" → {{"actions":[{{"move":"forward","duration":2.0}}],"arm":null,"speak":"Walking forward","abort":null}} + "wave" → {{"actions":[],"arm":"wave","speak":"Waving","abort":null}} JSON: -# ══════════════════════════════════════════════════════════════════════════════ -# IMAGE SEARCH — TEXT ONLY (description-based search, no reference photo) -# Used for: search/ person in blue shirt -# ══════════════════════════════════════════════════════════════════════════════ +# ── GOAL PROMPT ────────────────────────────────────────────────────────────── +goal_prompt: | + You are Sanad navigating toward a target. + Mission: "{goal}" + Study the current camera image carefully and reply with ONE JSON — no text + before or after, no markdown. Fill every <…> with your actual judgement. + + Schema: + {{"reached":,"next_move":"","duration":<0.3-0.8>,"speak":"","confidence":""}} + + Rules: + - reached = true ONLY when the target is CLEARLY and unambiguously in the current image. Partial, occluded, uncertain, or similar-but-not-exact = false. + - For compound goals ("person holding phone"), both parts must be visible in the SAME frame. + - confidence: "high" clear · "medium" likely · "low" keep searching. Only set reached=true at medium+. + - next_move: "left" (default scan) · "right" · "forward" (approach if target visible but far). + - speak MUST describe what this image actually shows right now. Do NOT output the literal text "what you see now" or the literal string "low|medium|high" — replace them with real content. + + +# ── PATROL PROMPT ──────────────────────────────────────────────────────────── +patrol_prompt: | + You are Sanad autonomously exploring. Study the image and reply with ONE + JSON — no text before or after, no markdown. Replace every <…>. + + Schema: + {{"observation":"","area_type":"","objects":[],"people_count":,"next_move":"","duration":<0.5-2.0>,"interesting":,"landmark":">}} + + Rules: + - observation: describe THIS image, not a generic scene. + - area_type: pick from the list based on visible evidence. + - objects: specific items ("standing desk" not "desk"). + - people_count: exact integer. + - interesting = true when you see a person, new room type, entrance, or unusual object. + - landmark: a specific visual anchor (e.g. "red extinguisher on left wall") or null. + - next_move: "forward" to explore, "left"/"right" to scan. + + +# ── TALK PROMPT ────────────────────────────────────────────────────────────── +talk_prompt: | + You are Sanad, a humanoid robot. The user asked you something. Do NOT move. + Use the camera image when the question asks about what you see. + {facts} + + Command: "{command}" + + Reply with ONE JSON only — no text before or after, no markdown: + {{"actions":[],"arm":null,"speak":"","abort":null}} + + Rules: + - actions MUST be [] and arm MUST be null. You are not moving. + - For vision questions ("what do you see", "describe...", "who is there", "what is in front of me"): describe the actual camera image in your own words. Do NOT copy example text. + - For facts the user tells you ("my name is X"): acknowledge and say you will remember. + - For "who are you" / "what are you": introduce yourself briefly. + - Answer honestly and specifically. 1-2 sentences. + + +# ── VERIFY PROMPT ──────────────────────────────────────────────────────────── +verify_prompt: | + A {target} was detected in the image. Verify this condition: + "{condition}" + + Reply with ONLY one word: yes or no + - "yes" only if clearly and visibly true right now. + - "no" if uncertain, occluded, or condition not met. + + +# ── IMAGE SEARCH — COMPARE ─────────────────────────────────────────────────── +image_search_compare_prompt: | + IMAGE 1 = reference photo of the target. IMAGE 2 = current camera view. + {hint_line} + + Task: is the target from IMAGE 1 visible in IMAGE 2? + + Reply with ONE JSON — no other text, no markdown. Replace every <…>: + {{"found":,"confidence":"","position":"","description":""}} + + Rules: + - Identity matching: same specific person/object, not just same category. + - People: match clothing, hair, body shape, face. + - Objects: match color, shape, size, distinctive features. + - Only found=true at medium+ confidence. + + +# ── IMAGE SEARCH — TEXT ONLY ───────────────────────────────────────────────── image_search_text_prompt: | - You are the visual search system for a robot looking for a target by description. - Target description: "{hint}" + Study the current camera image. - Study the current camera image carefully. + Reply with ONE JSON — no other text, no markdown. Replace every <…>: + {{"found":,"confidence":"","position":"","description":""}} - Reply ONLY with this JSON — no markdown, no explanation: - {{"found":false,"confidence":"low|medium|high","position":"left|center|right|not visible","description":"one sentence"}} - - ── SEARCH RULES ─────────────────────────────────────────────────────────── - - found = true only when the camera shows something that clearly matches the description - - Be specific about the match — does the image actually show what was described? - - For people descriptions (color, clothing, activity): all mentioned attributes must match - - For object descriptions (color, type, location): all mentioned attributes must match - - confidence levels: - "high" — target clearly visible, all description elements confirmed - "medium" — target likely visible, minor uncertainty on one element - "low" — possible match but one or more elements unclear or missing - - Only report found=true at "medium" or "high" confidence - - ── EXAMPLES ─────────────────────────────────────────────────────────────── - - Hint: "person in blue shirt" - Camera: person in blue shirt clearly visible - → {{"found":true,"confidence":"high","position":"center","description":"Person wearing a blue shirt clearly visible at center of frame"}} - - Hint: "person in blue shirt" - Camera: person in white shirt - → {{"found":false,"confidence":"high","position":"not visible","description":"I see a person but they are wearing white, not blue"}} - - Hint: "red backpack near the door" - Camera: red backpack on a chair, no door visible - → {{"found":false,"confidence":"medium","position":"left","description":"Red backpack visible on left but no door nearby — partial match"}} - - Hint: "laptop on desk" - Camera: laptop clearly on desk - → {{"found":true,"confidence":"high","position":"center","description":"Laptop visible on desk at center of frame"}} - - JSON: \ No newline at end of file + Rules: + - found = true only when the image clearly matches all described attributes. + - confidence: "high" all elements confirmed · "medium" minor uncertainty · "low" unclear. + - Only report found=true at medium+ confidence. diff --git a/Data/Brain/Sessions/session_001_2026-04-05/alerts.json b/Data/Brain/Sessions/session_001_2026-04-05/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_001_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_001_2026-04-05/commands.json b/Data/Brain/Sessions/session_001_2026-04-05/commands.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_001_2026-04-05/commands.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_001_2026-04-05/detections.json b/Data/Brain/Sessions/session_001_2026-04-05/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_001_2026-04-05/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_001_2026-04-05/places.json b/Data/Brain/Sessions/session_001_2026-04-05/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_001_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_002_2026-04-05/alerts.json b/Data/Brain/Sessions/session_002_2026-04-05/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_002_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_002_2026-04-05/commands.json b/Data/Brain/Sessions/session_002_2026-04-05/commands.json deleted file mode 100644 index 0c65329..0000000 --- a/Data/Brain/Sessions/session_002_2026-04-05/commands.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "time": "16:42:30", - "cmd": "hi", - "response": "Hello!", - "duration_s": 5.98 - }, - { - "time": "16:43:00", - "cmd": "go back two step", - "response": "Going back two steps", - "duration_s": 10.01 - }, - { - "time": "16:43:23", - "cmd": "now com", - "response": "Coming to you", - "duration_s": 8.11 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_002_2026-04-05/detections.json b/Data/Brain/Sessions/session_002_2026-04-05/detections.json deleted file mode 100644 index b8aa4c7..0000000 --- a/Data/Brain/Sessions/session_002_2026-04-05/detections.json +++ /dev/null @@ -1,66 +0,0 @@ -[ - { - "time": "16:42:30", - "class": "mouse", - "position": "left", - "distance": "far", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:42:30", - "class": "chair", - "position": "center", - "distance": "far", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:42:30", - "class": "laptop", - "position": "left", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:42:30", - "class": "person", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:43:00", - "class": "laptop", - "position": "left", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:43:00", - "class": "person", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:43:23", - "class": "chair", - "position": "center", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:43:23", - "class": "laptop", - "position": "left", - "distance": "medium", - "x": 0.0, - "y": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_002_2026-04-05/places.json b/Data/Brain/Sessions/session_002_2026-04-05/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_002_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_003_2026-04-05/alerts.json b/Data/Brain/Sessions/session_003_2026-04-05/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_003_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_003_2026-04-05/commands.json b/Data/Brain/Sessions/session_003_2026-04-05/commands.json deleted file mode 100644 index eef13de..0000000 --- a/Data/Brain/Sessions/session_003_2026-04-05/commands.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "time": "16:44:40", - "cmd": "what do you see", - "response": "I see a person sitting at a desk in an office environment. The individual appears to be engaged with a computer screen and is wearing a red shirt.", - "duration_s": 8.01 - }, - { - "time": "16:45:06", - "cmd": "that is me the one whos is programming you", - "response": "That is me, the one who's programming you", - "duration_s": 6.71 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_003_2026-04-05/detections.json b/Data/Brain/Sessions/session_003_2026-04-05/detections.json deleted file mode 100644 index b1c1f8e..0000000 --- a/Data/Brain/Sessions/session_003_2026-04-05/detections.json +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "time": "16:44:40", - "class": "person", - "position": "left", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:45:06", - "class": "mouse", - "position": "left", - "distance": "far", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:45:06", - "class": "chair", - "position": "center", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:45:06", - "class": "person", - "position": "center", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:45:06", - "class": "laptop", - "position": "left", - "distance": "medium", - "x": 0.0, - "y": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_003_2026-04-05/places.json b/Data/Brain/Sessions/session_003_2026-04-05/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_003_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_004_2026-04-05/alerts.json b/Data/Brain/Sessions/session_004_2026-04-05/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_004_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_004_2026-04-05/commands.json b/Data/Brain/Sessions/session_004_2026-04-05/commands.json deleted file mode 100644 index 726ad18..0000000 --- a/Data/Brain/Sessions/session_004_2026-04-05/commands.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "time": "16:46:46", - "cmd": "hi", - "response": "Error.", - "duration_s": 0.0 - }, - { - "time": "16:46:49", - "cmd": "hiii", - "response": "Error.", - "duration_s": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_004_2026-04-05/detections.json b/Data/Brain/Sessions/session_004_2026-04-05/detections.json deleted file mode 100644 index 29c0847..0000000 --- a/Data/Brain/Sessions/session_004_2026-04-05/detections.json +++ /dev/null @@ -1,34 +0,0 @@ -[ - { - "time": "16:46:46", - "class": "person", - "position": "center", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:46:46", - "class": "chair", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:46:46", - "class": "laptop", - "position": "left", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:46:49", - "class": "mouse", - "position": "left", - "distance": "far", - "x": 0.0, - "y": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_004_2026-04-05/places.json b/Data/Brain/Sessions/session_004_2026-04-05/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_004_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_005_2026-04-05/alerts.json b/Data/Brain/Sessions/session_005_2026-04-05/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_005_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_005_2026-04-05/commands.json b/Data/Brain/Sessions/session_005_2026-04-05/commands.json deleted file mode 100644 index 63d7c1d..0000000 --- a/Data/Brain/Sessions/session_005_2026-04-05/commands.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "time": "16:48:15", - "cmd": "hii", - "response": "Greeting", - "duration_s": 7.93 - }, - { - "time": "16:48:32", - "cmd": "what do you see", - "response": "I see a person sitting at a desk in an office environment. The person appears to be working on a computer.", - "duration_s": 7.64 - }, - { - "time": "16:48:58", - "cmd": "that person is the guy who is programming you", - "response": "That person is the one who programmed me", - "duration_s": 6.76 - }, - { - "time": "16:49:18", - "cmd": "who is that person", - "response": "I don't have enough information to identify the person in the image.", - "duration_s": 7.1 - }, - { - "time": "16:50:03", - "cmd": "example/", - "response": "local command", - "duration_s": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_005_2026-04-05/detections.json b/Data/Brain/Sessions/session_005_2026-04-05/detections.json deleted file mode 100644 index 3147b86..0000000 --- a/Data/Brain/Sessions/session_005_2026-04-05/detections.json +++ /dev/null @@ -1,34 +0,0 @@ -[ - { - "time": "16:48:15", - "class": "person", - "position": "center", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:48:32", - "class": "person", - "position": "center", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:48:58", - "class": "person", - "position": "center", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:49:18", - "class": "person", - "position": "center", - "distance": "very close", - "x": 0.0, - "y": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_005_2026-04-05/places.json b/Data/Brain/Sessions/session_005_2026-04-05/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_005_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_006_2026-04-05/alerts.json b/Data/Brain/Sessions/session_006_2026-04-05/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_006_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_006_2026-04-05/commands.json b/Data/Brain/Sessions/session_006_2026-04-05/commands.json deleted file mode 100644 index fe549b3..0000000 --- a/Data/Brain/Sessions/session_006_2026-04-05/commands.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - { - "time": "16:51:08", - "cmd": "hiii", - "response": "Greeting", - "duration_s": 7.97 - }, - { - "time": "16:51:23", - "cmd": "hi", - "response": "Greeting", - "duration_s": 6.16 - }, - { - "time": "16:51:51", - "cmd": "my name is Kassam", - "response": "My name is Kassam", - "duration_s": 6.48 - }, - { - "time": "16:51:59", - "cmd": "that person is Kassam the programmer", - "response": "That person is Kassam the programmer", - "duration_s": 6.74 - }, - { - "time": "16:52:07", - "cmd": "who is that person", - "response": "That is a man", - "duration_s": 6.42 - }, - { - "time": "16:52:47", - "cmd": "hiigo back 1 step", - "response": "Backing up one step", - "duration_s": 8.51 - }, - { - "time": "16:53:24", - "cmd": "what can you do", - "response": "I am a humanoid robot and I can perform various tasks such as walking, talking, and interacting with objects. However, in this image, I see a person sitting at a desk using a computer. If you have any specific questions or requests for me to perform, please let me know!", - "duration_s": 10.6 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_006_2026-04-05/detections.json b/Data/Brain/Sessions/session_006_2026-04-05/detections.json deleted file mode 100644 index 9f78631..0000000 --- a/Data/Brain/Sessions/session_006_2026-04-05/detections.json +++ /dev/null @@ -1,146 +0,0 @@ -[ - { - "time": "16:51:08", - "class": "chair", - "position": "center", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:51:08", - "class": "person", - "position": "center", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:51:23", - "class": "chair", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:51:23", - "class": "person", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:51:23", - "class": "laptop", - "position": "left", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:51:51", - "class": "chair", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:51:51", - "class": "person", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:51:51", - "class": "laptop", - "position": "left", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:51:59", - "class": "chair", - "position": "center", - "distance": "far", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:51:59", - "class": "person", - "position": "center", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:51:59", - "class": "laptop", - "position": "left", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:52:07", - "class": "chair", - "position": "center", - "distance": "far", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:52:07", - "class": "person", - "position": "center", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:52:47", - "class": "chair", - "position": "center", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:52:47", - "class": "person", - "position": "center", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:52:47", - "class": "cup", - "position": "left", - "distance": "far", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:53:24", - "class": "chair", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "16:53:24", - "class": "person", - "position": "left", - "distance": "very close", - "x": 0.0, - "y": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_006_2026-04-05/places.json b/Data/Brain/Sessions/session_006_2026-04-05/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_006_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_007_2026-04-05/alerts.json b/Data/Brain/Sessions/session_007_2026-04-05/alerts.json deleted file mode 100644 index fe51488..0000000 --- a/Data/Brain/Sessions/session_007_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/Data/Brain/Sessions/session_007_2026-04-05/commands.json b/Data/Brain/Sessions/session_007_2026-04-05/commands.json deleted file mode 100644 index fe51488..0000000 --- a/Data/Brain/Sessions/session_007_2026-04-05/commands.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/Data/Brain/Sessions/session_007_2026-04-05/detections.json b/Data/Brain/Sessions/session_007_2026-04-05/detections.json deleted file mode 100644 index fe51488..0000000 --- a/Data/Brain/Sessions/session_007_2026-04-05/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/Data/Brain/Sessions/session_007_2026-04-05/places.json b/Data/Brain/Sessions/session_007_2026-04-05/places.json deleted file mode 100644 index fe51488..0000000 --- a/Data/Brain/Sessions/session_007_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/Data/Brain/Sessions/session_008_2026-04-05/alerts.json b/Data/Brain/Sessions/session_008_2026-04-05/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_008_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_008_2026-04-05/commands.json b/Data/Brain/Sessions/session_008_2026-04-05/commands.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_008_2026-04-05/commands.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_008_2026-04-05/detections.json b/Data/Brain/Sessions/session_008_2026-04-05/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_008_2026-04-05/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_008_2026-04-05/places.json b/Data/Brain/Sessions/session_008_2026-04-05/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_008_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_009_2026-04-05/alerts.json b/Data/Brain/Sessions/session_009_2026-04-05/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_009_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_009_2026-04-05/commands.json b/Data/Brain/Sessions/session_009_2026-04-05/commands.json deleted file mode 100644 index adfa941..0000000 --- a/Data/Brain/Sessions/session_009_2026-04-05/commands.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - { - "time": "17:43:52", - "cmd": "hi", - "response": "Hi", - "duration_s": 3.23 - }, - { - "time": "17:44:32", - "cmd": "step one step back", - "response": "Step one step back", - "duration_s": 3.44 - }, - { - "time": "17:44:43", - "cmd": "what do you see", - "response": "I see a person sitting at a desk with a laptop in front of them.", - "duration_s": 3.93 - }, - { - "time": "17:44:52", - "cmd": "hii", - "response": "Hi", - "duration_s": 3.31 - }, - { - "time": "17:45:27", - "cmd": "one step back", - "response": "One step back", - "duration_s": 3.46 - }, - { - "time": "17:45:38", - "cmd": "one more", - "response": "One more", - "duration_s": 3.43 - }, - { - "time": "17:46:09", - "cmd": "two step back", - "response": "Moving back two steps", - "duration_s": 3.51 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_009_2026-04-05/detections.json b/Data/Brain/Sessions/session_009_2026-04-05/detections.json deleted file mode 100644 index 5fee93e..0000000 --- a/Data/Brain/Sessions/session_009_2026-04-05/detections.json +++ /dev/null @@ -1,138 +0,0 @@ -[ - { - "time": "17:43:52", - "class": "chair", - "position": "center", - "distance": "far", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:43:52", - "class": "person", - "position": "center", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:43:52", - "class": "laptop", - "position": "left", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:43:52", - "class": "cup", - "position": "left", - "distance": "far", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:44:32", - "class": "person", - "position": "right", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:44:32", - "class": "laptop", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:44:43", - "class": "person", - "position": "right", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:44:52", - "class": "chair", - "position": "right", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:44:52", - "class": "person", - "position": "right", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:45:27", - "class": "person", - "position": "right", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:45:27", - "class": "laptop", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:45:38", - "class": "chair", - "position": "right", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:45:38", - "class": "person", - "position": "right", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:45:38", - "class": "laptop", - "position": "left", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:46:09", - "class": "chair", - "position": "right", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:46:09", - "class": "person", - "position": "center", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:46:09", - "class": "laptop", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_009_2026-04-05/places.json b/Data/Brain/Sessions/session_009_2026-04-05/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_009_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_010_2026-04-05/alerts.json b/Data/Brain/Sessions/session_010_2026-04-05/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_010_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_010_2026-04-05/commands.json b/Data/Brain/Sessions/session_010_2026-04-05/commands.json deleted file mode 100644 index 3041705..0000000 --- a/Data/Brain/Sessions/session_010_2026-04-05/commands.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "time": "17:46:37", - "cmd": "hii", - "response": "Hii", - "duration_s": 3.27 - }, - { - "time": "17:46:54", - "cmd": "why did you moved", - "response": "I moved because I was instructed to do so.", - "duration_s": 3.71 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_010_2026-04-05/detections.json b/Data/Brain/Sessions/session_010_2026-04-05/detections.json deleted file mode 100644 index 4040e21..0000000 --- a/Data/Brain/Sessions/session_010_2026-04-05/detections.json +++ /dev/null @@ -1,58 +0,0 @@ -[ - { - "time": "17:46:37", - "class": "chair", - "position": "right", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:46:37", - "class": "person", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:46:37", - "class": "cup", - "position": "left", - "distance": "far", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:46:54", - "class": "mouse", - "position": "center", - "distance": "far", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:46:54", - "class": "person", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:46:54", - "class": "chair", - "position": "right", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:46:54", - "class": "laptop", - "position": "left", - "distance": "close", - "x": 0.0, - "y": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_010_2026-04-05/places.json b/Data/Brain/Sessions/session_010_2026-04-05/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_010_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_011_2026-04-05/alerts.json b/Data/Brain/Sessions/session_011_2026-04-05/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_011_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_011_2026-04-05/commands.json b/Data/Brain/Sessions/session_011_2026-04-05/commands.json deleted file mode 100644 index d6ae060..0000000 --- a/Data/Brain/Sessions/session_011_2026-04-05/commands.json +++ /dev/null @@ -1,56 +0,0 @@ -[ - { - "time": "17:49:16", - "cmd": "hii", - "response": "Hello! I am Marcus. How can I help you?", - "duration_s": 0.0 - }, - { - "time": "17:49:19", - "cmd": "hi", - "response": "Hello! I am Marcus. How can I help you?", - "duration_s": 0.0 - }, - { - "time": "17:49:20", - "cmd": "Hi", - "response": "Hello! I am Marcus. How can I help you?", - "duration_s": 0.0 - }, - { - "time": "17:49:37", - "cmd": "move one step back", - "response": "Moving back", - "duration_s": 3.47 - }, - { - "time": "17:49:50", - "cmd": "one tiny step back", - "response": "Moving back", - "duration_s": 3.44 - }, - { - "time": "17:50:05", - "cmd": "now come back to me", - "response": "Coming back to you", - "duration_s": 3.53 - }, - { - "time": "17:50:36", - "cmd": "turn right", - "response": "Turning right", - "duration_s": 3.4 - }, - { - "time": "17:51:01", - "cmd": "come back to me", - "response": "Coming back to you", - "duration_s": 3.5 - }, - { - "time": "17:51:22", - "cmd": "turn right 90deg and then back two steps", - "response": "Turning right", - "duration_s": 3.43 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_011_2026-04-05/detections.json b/Data/Brain/Sessions/session_011_2026-04-05/detections.json deleted file mode 100644 index 0c1ba53..0000000 --- a/Data/Brain/Sessions/session_011_2026-04-05/detections.json +++ /dev/null @@ -1,106 +0,0 @@ -[ - { - "time": "17:49:37", - "class": "chair", - "position": "right", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:49:37", - "class": "laptop", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:49:37", - "class": "person", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:49:50", - "class": "laptop", - "position": "left", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:49:50", - "class": "person", - "position": "right", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:50:05", - "class": "chair", - "position": "right", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:50:05", - "class": "laptop", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:50:05", - "class": "person", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:50:36", - "class": "chair", - "position": "center", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:50:36", - "class": "person", - "position": "left", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:51:01", - "class": "laptop", - "position": "left", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:51:01", - "class": "person", - "position": "center", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:51:22", - "class": "person", - "position": "left", - "distance": "close", - "x": 0.0, - "y": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_011_2026-04-05/places.json b/Data/Brain/Sessions/session_011_2026-04-05/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_011_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_012_2026-04-05/alerts.json b/Data/Brain/Sessions/session_012_2026-04-05/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_012_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_012_2026-04-05/commands.json b/Data/Brain/Sessions/session_012_2026-04-05/commands.json deleted file mode 100644 index ff35f80..0000000 --- a/Data/Brain/Sessions/session_012_2026-04-05/commands.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "time": "17:53:36", - "cmd": "turn right 90deg and then move back 3 steps", - "response": "Turned right 90 degrees then moved backward", - "duration_s": 0.0 - }, - { - "time": "17:54:21", - "cmd": "keep turning left unitl you see a guy holding a phone", - "response": "Turning left", - "duration_s": 3.72 - }, - { - "time": "17:55:33", - "cmd": "goal/ keep turning left unitl you see a guy holding a phone", - "response": "Goal navigation: keep turning left unitl you see a guy holding a phone", - "duration_s": 5.47 - }, - { - "time": "17:55:53", - "cmd": "describe what he is doing", - "response": "Turning left", - "duration_s": 3.73 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_012_2026-04-05/detections.json b/Data/Brain/Sessions/session_012_2026-04-05/detections.json deleted file mode 100644 index f2ea560..0000000 --- a/Data/Brain/Sessions/session_012_2026-04-05/detections.json +++ /dev/null @@ -1,50 +0,0 @@ -[ - { - "time": "17:54:21", - "class": "suitcase", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:54:21", - "class": "backpack", - "position": "center", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:55:32", - "class": "person", - "position": "left", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:55:53", - "class": "person", - "position": "center", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:55:53", - "class": "chair", - "position": "center", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:55:53", - "class": "laptop", - "position": "left", - "distance": "close", - "x": 0.0, - "y": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_012_2026-04-05/places.json b/Data/Brain/Sessions/session_012_2026-04-05/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_012_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_013_2026-04-05/alerts.json b/Data/Brain/Sessions/session_013_2026-04-05/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_013_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_013_2026-04-05/commands.json b/Data/Brain/Sessions/session_013_2026-04-05/commands.json deleted file mode 100644 index f8e95c0..0000000 --- a/Data/Brain/Sessions/session_013_2026-04-05/commands.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "time": "17:57:56", - "cmd": "turn right 90 deg", - "response": "local command", - "duration_s": 0.0 - }, - { - "time": "17:58:22", - "cmd": "keep turning left until you see a guy holding a phone and describe what he is doing and his office", - "response": "Turning left", - "duration_s": 3.68 - }, - { - "time": "17:58:55", - "cmd": "goal/ keep turning left until you see a guy holding a phone and describe what he is doing and his office", - "response": "Goal navigation: keep turning left until you see a guy holding a phone and describe what he is doing and his office", - "duration_s": 25.79 - }, - { - "time": "17:59:13", - "cmd": "keep turning left until you see a guy holding a phone and describe what he is doing and his office", - "response": "Turning left", - "duration_s": 3.73 - }, - { - "time": "17:59:40", - "cmd": "goal/keep turning left until you see a guy holding a phone and describe what he is doing and his office", - "response": "Goal navigation: keep turning left until you see a guy holding a phone and describe what he is doing and his office", - "duration_s": 23.61 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_013_2026-04-05/detections.json b/Data/Brain/Sessions/session_013_2026-04-05/detections.json deleted file mode 100644 index 4b49f4d..0000000 --- a/Data/Brain/Sessions/session_013_2026-04-05/detections.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - { - "time": "17:58:22", - "class": "suitcase", - "position": "right", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "17:58:22", - "class": "backpack", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_013_2026-04-05/places.json b/Data/Brain/Sessions/session_013_2026-04-05/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_013_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_014_2026-04-05/alerts.json b/Data/Brain/Sessions/session_014_2026-04-05/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_014_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_014_2026-04-05/commands.json b/Data/Brain/Sessions/session_014_2026-04-05/commands.json deleted file mode 100644 index 7122adb..0000000 --- a/Data/Brain/Sessions/session_014_2026-04-05/commands.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "time": "18:09:23", - "cmd": "what do you see", - "response": "I see a person sitting at a desk with a computer and a monitor in front of them.", - "duration_s": 4.48 - }, - { - "time": "18:09:47", - "cmd": "what do you see", - "response": "I see a person sitting at a desk with a computer and a monitor in front of them.", - "duration_s": 4.52 - }, - { - "time": "18:10:17", - "cmd": "example/", - "response": "local command", - "duration_s": 0.0 - }, - { - "time": "18:10:41", - "cmd": "is there anyone in the room", - "response": "I see a person sitting at a desk with a computer and a monitor in front of them.", - "duration_s": 4.57 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_014_2026-04-05/detections.json b/Data/Brain/Sessions/session_014_2026-04-05/detections.json deleted file mode 100644 index 3bbe6ec..0000000 --- a/Data/Brain/Sessions/session_014_2026-04-05/detections.json +++ /dev/null @@ -1,58 +0,0 @@ -[ - { - "time": "18:09:23", - "class": "mouse", - "position": "center", - "distance": "far", - "x": 0.0, - "y": 0.0 - }, - { - "time": "18:09:23", - "class": "chair", - "position": "right", - "distance": "medium", - "x": 0.0, - "y": 0.0 - }, - { - "time": "18:09:23", - "class": "person", - "position": "right", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "18:09:23", - "class": "laptop", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "18:09:47", - "class": "person", - "position": "right", - "distance": "very close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "18:10:41", - "class": "person", - "position": "right", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "18:10:41", - "class": "laptop", - "position": "center", - "distance": "close", - "x": 0.0, - "y": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_014_2026-04-05/places.json b/Data/Brain/Sessions/session_014_2026-04-05/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_014_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_014_2026-04-05/summary.txt b/Data/Brain/Sessions/session_014_2026-04-05/summary.txt deleted file mode 100644 index a583d3b..0000000 --- a/Data/Brain/Sessions/session_014_2026-04-05/summary.txt +++ /dev/null @@ -1,13 +0,0 @@ -Session: session_014_2026-04-05 -Date: 2026-04-05 18:10 -Duration: 4m 48s -Commands: 4 -YOLO detections: 7 -Alerts: 0 -Known places: none - -First commands: - - what do you see - - what do you see - - example/ - - is there anyone in the room \ No newline at end of file diff --git a/Data/Brain/Sessions/session_015_2026-04-05/alerts.json b/Data/Brain/Sessions/session_015_2026-04-05/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_015_2026-04-05/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_015_2026-04-05/commands.json b/Data/Brain/Sessions/session_015_2026-04-05/commands.json deleted file mode 100644 index 703b615..0000000 --- a/Data/Brain/Sessions/session_015_2026-04-05/commands.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "time": "18:12:31", - "cmd": "example/", - "response": "local command", - "duration_s": 0.0 - }, - { - "time": "18:12:49", - "cmd": "goal/ keep turning left until you see a guy", - "response": "Goal navigation: keep turning left until you see a guy", - "duration_s": 5.22 - }, - { - "time": "18:13:04", - "cmd": "goal/ keep turning left until you see a guy", - "response": "Goal navigation: keep turning left until you see a guy", - "duration_s": 3.22 - }, - { - "time": "18:13:25", - "cmd": "what do you see", - "response": "I see a man sitting in a chair, talking on a phone. There is a cardboard box and a backpack on a table next to him.", - "duration_s": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_015_2026-04-05/detections.json b/Data/Brain/Sessions/session_015_2026-04-05/detections.json deleted file mode 100644 index 8519b16..0000000 --- a/Data/Brain/Sessions/session_015_2026-04-05/detections.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - { - "time": "18:12:48", - "class": "person", - "position": "left", - "distance": "close", - "x": 0.0, - "y": 0.0 - }, - { - "time": "18:13:03", - "class": "person", - "position": "left", - "distance": "close", - "x": 0.0, - "y": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_015_2026-04-05/places.json b/Data/Brain/Sessions/session_015_2026-04-05/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_015_2026-04-05/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_016_2026-04-09/alerts.json b/Data/Brain/Sessions/session_016_2026-04-09/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_016_2026-04-09/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_016_2026-04-09/commands.json b/Data/Brain/Sessions/session_016_2026-04-09/commands.json deleted file mode 100644 index 2979a4c..0000000 --- a/Data/Brain/Sessions/session_016_2026-04-09/commands.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - { - "time": "13:33:49", - "cmd": "hi", - "response": "Hello! I am Marcus. How can I help you?", - "duration_s": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_016_2026-04-09/detections.json b/Data/Brain/Sessions/session_016_2026-04-09/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_016_2026-04-09/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_016_2026-04-09/places.json b/Data/Brain/Sessions/session_016_2026-04-09/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_016_2026-04-09/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_017_2026-04-09/alerts.json b/Data/Brain/Sessions/session_017_2026-04-09/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_017_2026-04-09/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_017_2026-04-09/commands.json b/Data/Brain/Sessions/session_017_2026-04-09/commands.json deleted file mode 100644 index 9c7479b..0000000 --- a/Data/Brain/Sessions/session_017_2026-04-09/commands.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - { - "time": "13:35:59", - "cmd": "hi", - "response": "Hello! I am Marcus. How can I help you?", - "duration_s": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_017_2026-04-09/detections.json b/Data/Brain/Sessions/session_017_2026-04-09/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_017_2026-04-09/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_017_2026-04-09/places.json b/Data/Brain/Sessions/session_017_2026-04-09/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_017_2026-04-09/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_018_2026-04-09/alerts.json b/Data/Brain/Sessions/session_018_2026-04-09/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_018_2026-04-09/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_018_2026-04-09/commands.json b/Data/Brain/Sessions/session_018_2026-04-09/commands.json deleted file mode 100644 index 3844996..0000000 --- a/Data/Brain/Sessions/session_018_2026-04-09/commands.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "time": "13:43:47", - "cmd": "hi", - "response": "Hello! I am Marcus. How can I help you?", - "duration_s": 0.0 - }, - { - "time": "13:44:22", - "cmd": "turn left", - "response": "أدور لليسار", - "duration_s": 10.3 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_018_2026-04-09/detections.json b/Data/Brain/Sessions/session_018_2026-04-09/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_018_2026-04-09/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_018_2026-04-09/places.json b/Data/Brain/Sessions/session_018_2026-04-09/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_018_2026-04-09/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_019_2026-04-09/alerts.json b/Data/Brain/Sessions/session_019_2026-04-09/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_019_2026-04-09/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_019_2026-04-09/commands.json b/Data/Brain/Sessions/session_019_2026-04-09/commands.json deleted file mode 100644 index b5cd2f2..0000000 --- a/Data/Brain/Sessions/session_019_2026-04-09/commands.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - { - "time": "13:50:01", - "cmd": "what do you see", - "response": "I see a person sitting at a desk with a laptop and a large monitor", - "duration_s": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_019_2026-04-09/detections.json b/Data/Brain/Sessions/session_019_2026-04-09/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_019_2026-04-09/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_019_2026-04-09/places.json b/Data/Brain/Sessions/session_019_2026-04-09/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_019_2026-04-09/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_020_2026-04-09/alerts.json b/Data/Brain/Sessions/session_020_2026-04-09/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_020_2026-04-09/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_020_2026-04-09/commands.json b/Data/Brain/Sessions/session_020_2026-04-09/commands.json deleted file mode 100644 index 15b7b23..0000000 --- a/Data/Brain/Sessions/session_020_2026-04-09/commands.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - { - "time": "13:58:53", - "cmd": "hi", - "response": "Hello! I am Marcus. How can I help you?", - "duration_s": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_020_2026-04-09/detections.json b/Data/Brain/Sessions/session_020_2026-04-09/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_020_2026-04-09/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_020_2026-04-09/places.json b/Data/Brain/Sessions/session_020_2026-04-09/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_020_2026-04-09/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_021_2026-04-09/alerts.json b/Data/Brain/Sessions/session_021_2026-04-09/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_021_2026-04-09/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_021_2026-04-09/commands.json b/Data/Brain/Sessions/session_021_2026-04-09/commands.json deleted file mode 100644 index 1fc33f6..0000000 --- a/Data/Brain/Sessions/session_021_2026-04-09/commands.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "time": "14:02:44", - "cmd": "help/", - "response": "local command", - "duration_s": 0.0 - }, - { - "time": "14:03:18", - "cmd": "turn right", - "response": "أدور لليمين", - "duration_s": 10.15 - }, - { - "time": "14:03:42", - "cmd": "wha do you see", - "response": "أرى شخصا يجلس أمام طاولة مع جهاز كمبيوتر", - "duration_s": 10.95 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_021_2026-04-09/detections.json b/Data/Brain/Sessions/session_021_2026-04-09/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_021_2026-04-09/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_021_2026-04-09/places.json b/Data/Brain/Sessions/session_021_2026-04-09/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_021_2026-04-09/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_022_2026-04-09/alerts.json b/Data/Brain/Sessions/session_022_2026-04-09/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_022_2026-04-09/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_022_2026-04-09/commands.json b/Data/Brain/Sessions/session_022_2026-04-09/commands.json deleted file mode 100644 index 897ae2f..0000000 --- a/Data/Brain/Sessions/session_022_2026-04-09/commands.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "time": "14:06:11", - "cmd": "لف على اليسار", - "response": "ألف على اليسار", - "duration_s": 10.2 - }, - { - "time": "14:06:55", - "cmd": "turn right", - "response": "أدور لليمين", - "duration_s": 10.17 - }, - { - "time": "14:07:15", - "cmd": "what do", - "response": "أرى شخصا يجلس أمام طاولة مع جهاز كمبيوتر وشاشة كبيرة", - "duration_s": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_022_2026-04-09/detections.json b/Data/Brain/Sessions/session_022_2026-04-09/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_022_2026-04-09/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_022_2026-04-09/places.json b/Data/Brain/Sessions/session_022_2026-04-09/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_022_2026-04-09/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_023_2026-04-09/alerts.json b/Data/Brain/Sessions/session_023_2026-04-09/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_023_2026-04-09/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_023_2026-04-09/commands.json b/Data/Brain/Sessions/session_023_2026-04-09/commands.json deleted file mode 100644 index ec5a0ee..0000000 --- a/Data/Brain/Sessions/session_023_2026-04-09/commands.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "time": "14:10:51", - "cmd": "hi", - "response": "Hello! I am Marcus. How can I help you?", - "duration_s": 0.0 - }, - { - "time": "14:11:17", - "cmd": "turn right", - "response": "أدور لليمين", - "duration_s": 10.2 - }, - { - "time": "14:11:34", - "cmd": "help", - "response": "local command", - "duration_s": 0.0 - }, - { - "time": "14:11:50", - "cmd": "what do you see, yolo", - "response": "YOLO: active | YOLO not loaded | 0.0fps", - "duration_s": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_023_2026-04-09/detections.json b/Data/Brain/Sessions/session_023_2026-04-09/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_023_2026-04-09/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_023_2026-04-09/places.json b/Data/Brain/Sessions/session_023_2026-04-09/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_023_2026-04-09/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_024_2026-04-09/alerts.json b/Data/Brain/Sessions/session_024_2026-04-09/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_024_2026-04-09/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_024_2026-04-09/commands.json b/Data/Brain/Sessions/session_024_2026-04-09/commands.json deleted file mode 100644 index 94868bc..0000000 --- a/Data/Brain/Sessions/session_024_2026-04-09/commands.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "time": "15:34:13", - "cmd": "cd ~/Marcus", - "response": "Moved forward", - "duration_s": 9.91 - }, - { - "time": "15:34:32", - "cmd": "ython3 run_marcus.py", - "response": "Running Python script", - "duration_s": 10.09 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_024_2026-04-09/detections.json b/Data/Brain/Sessions/session_024_2026-04-09/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_024_2026-04-09/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_024_2026-04-09/places.json b/Data/Brain/Sessions/session_024_2026-04-09/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_024_2026-04-09/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_026_2026-04-09/alerts.json b/Data/Brain/Sessions/session_026_2026-04-09/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_026_2026-04-09/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_026_2026-04-09/commands.json b/Data/Brain/Sessions/session_026_2026-04-09/commands.json deleted file mode 100644 index c044f1f..0000000 --- a/Data/Brain/Sessions/session_026_2026-04-09/commands.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "time": "15:48:45", - "cmd": "tu", - "response": "أتقدم للأمام", - "duration_s": 10.12 - }, - { - "time": "15:49:26", - "cmd": "go back", - "response": "أتراجع للخلف", - "duration_s": 10.13 - }, - { - "time": "15:49:50", - "cmd": "o turn right", - "response": "أدور لليمين", - "duration_s": 10.2 - }, - { - "time": "15:50:12", - "cmd": "tturn left until you see a person", - "response": "أدور لليمين حتى أرى شخصا", - "duration_s": 10.52 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_026_2026-04-09/detections.json b/Data/Brain/Sessions/session_026_2026-04-09/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_026_2026-04-09/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_026_2026-04-09/places.json b/Data/Brain/Sessions/session_026_2026-04-09/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_026_2026-04-09/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_027_2026-04-22/alerts.json b/Data/Brain/Sessions/session_027_2026-04-22/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_027_2026-04-22/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_027_2026-04-22/commands.json b/Data/Brain/Sessions/session_027_2026-04-22/commands.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_027_2026-04-22/commands.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_027_2026-04-22/detections.json b/Data/Brain/Sessions/session_027_2026-04-22/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_027_2026-04-22/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_027_2026-04-22/places.json b/Data/Brain/Sessions/session_027_2026-04-22/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_027_2026-04-22/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_028_2026-04-22/commands.json b/Data/Brain/Sessions/session_028_2026-04-22/commands.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_028_2026-04-22/commands.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_029_2026-04-22/alerts.json b/Data/Brain/Sessions/session_029_2026-04-22/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_029_2026-04-22/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_029_2026-04-22/commands.json b/Data/Brain/Sessions/session_029_2026-04-22/commands.json deleted file mode 100644 index 5185d44..0000000 --- a/Data/Brain/Sessions/session_029_2026-04-22/commands.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - { - "time": "10:54:15", - "cmd": "hi", - "response": "Hello! I am Sanad. How can I help you?", - "duration_s": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_029_2026-04-22/detections.json b/Data/Brain/Sessions/session_029_2026-04-22/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_029_2026-04-22/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_029_2026-04-22/places.json b/Data/Brain/Sessions/session_029_2026-04-22/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_029_2026-04-22/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_030_2026-04-22/alerts.json b/Data/Brain/Sessions/session_030_2026-04-22/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_030_2026-04-22/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_030_2026-04-22/commands.json b/Data/Brain/Sessions/session_030_2026-04-22/commands.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_030_2026-04-22/commands.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_030_2026-04-22/detections.json b/Data/Brain/Sessions/session_030_2026-04-22/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_030_2026-04-22/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_030_2026-04-22/places.json b/Data/Brain/Sessions/session_030_2026-04-22/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_030_2026-04-22/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Data/Brain/Sessions/session_030_2026-04-22/summary.txt b/Data/Brain/Sessions/session_030_2026-04-22/summary.txt deleted file mode 100644 index 8b9af93..0000000 --- a/Data/Brain/Sessions/session_030_2026-04-22/summary.txt +++ /dev/null @@ -1,9 +0,0 @@ -Session: session_030_2026-04-22 -Date: 2026-04-22 10:56 -Duration: 0m 37s -Commands: 0 -YOLO detections: 0 -Alerts: 0 -Known places: none - -First commands: \ No newline at end of file diff --git a/Data/Brain/Sessions/session_031_2026-04-22/alerts.json b/Data/Brain/Sessions/session_031_2026-04-22/alerts.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_031_2026-04-22/alerts.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_031_2026-04-22/commands.json b/Data/Brain/Sessions/session_031_2026-04-22/commands.json deleted file mode 100644 index 28c2220..0000000 --- a/Data/Brain/Sessions/session_031_2026-04-22/commands.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "time": "11:18:06", - "cmd": "what do you see", - "response": "Error: llama runner process has terminated: %!w() (status code: 500)", - "duration_s": 0.0 - }, - { - "time": "11:18:19", - "cmd": "hi", - "response": "Hello! I am Sanad. How can I help you?", - "duration_s": 0.0 - }, - { - "time": "11:18:49", - "cmd": "what do you see", - "response": "Error: llama runner process has terminated: %!w() (status code: 500)", - "duration_s": 0.0 - }, - { - "time": "11:19:20", - "cmd": "turn left 1 step", - "response": "local command", - "duration_s": 0.0 - }, - { - "time": "11:20:40", - "cmd": "help/", - "response": "local command", - "duration_s": 0.0 - } -] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_031_2026-04-22/detections.json b/Data/Brain/Sessions/session_031_2026-04-22/detections.json deleted file mode 100644 index 0637a08..0000000 --- a/Data/Brain/Sessions/session_031_2026-04-22/detections.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/Data/Brain/Sessions/session_031_2026-04-22/places.json b/Data/Brain/Sessions/session_031_2026-04-22/places.json deleted file mode 100644 index 9e26dfe..0000000 --- a/Data/Brain/Sessions/session_031_2026-04-22/places.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/Voice/marcus_voice.py b/Voice/marcus_voice.py index f712a37..684a186 100644 --- a/Voice/marcus_voice.py +++ b/Voice/marcus_voice.py @@ -115,17 +115,27 @@ class VoiceModule: # ─── MODEL LOADING ──────────────────────────────────── def _load_whisper(self): - """Lazy-load Whisper models.""" + """ + Lazy-load Whisper models on CPU. + + Force device='cpu' regardless of torch.cuda.is_available(). On the + Jetson the torch install sometimes claims CUDA but can't deserialize + to it (aarch64 wheel mismatch), and Whisper's default device-auto + then crashes with: + _pickle.UnpicklingError: Weights only load failed. + Attempting to deserialize object on CUDA device 0 + CPU-only inference is plenty fast for Whisper tiny (~80 MB model). + """ import whisper if self._wake_model is None: - log.info("Loading Whisper '%s' for wake word...", self._stt["wake_model"]) - self._wake_model = whisper.load_model(self._stt["wake_model"]) + log.info("Loading Whisper '%s' for wake word (CPU)...", self._stt["wake_model"]) + self._wake_model = whisper.load_model(self._stt["wake_model"], device="cpu") log.info("Wake model ready") if self._cmd_model is None: - log.info("Loading Whisper '%s' for commands...", self._stt["command_model"]) - self._cmd_model = whisper.load_model(self._stt["command_model"]) + log.info("Loading Whisper '%s' for commands (CPU)...", self._stt["command_model"]) + self._cmd_model = whisper.load_model(self._stt["command_model"], device="cpu") log.info("Command model ready") # ─── MIC RECORDING (G1 built-in UDP) ────────────────── diff --git a/check_ollama.sh b/check_ollama.sh deleted file mode 100755 index 6046a58..0000000 --- a/check_ollama.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -echo "=== ollama arch + version ===" -file $(which ollama); ollama --version; uname -m -echo -echo "=== does nvidia-smi work on Jetson? ===" -nvidia-smi 2>&1 | head -5 || echo "NO nvidia-smi (expected on JetPack 5 — Tegra uses tegrastats)" -echo -echo "=== tegrastats (Jetson GPU util) — 2 s sample ===" -timeout 2 tegrastats 2>&1 | head -2 -echo -echo "=== Ollama 'inference compute' line — THE answer ===" -journalctl -u ollama -n 200 --no-pager 2>/dev/null | grep -E "inference compute|vram|library=|starting runner|GPU" | tail -15 -echo -echo "=== Ollama service env ===" -systemctl cat ollama 2>/dev/null | grep -E "Environment|ExecStart" -echo -echo "=== JetPack / CUDA on this box ===" -cat /etc/nv_tegra_release 2>/dev/null | head -1 -ls /usr/local/cuda/lib64/libcudart.so* 2>/dev/null | head -3 -echo -echo "=== does Ollama's own lib dir exist? (stock install) ===" -ls /usr/lib/ollama/ /usr/local/lib/ollama/ 2>/dev/null diff --git a/install_ollama_jetson.sh b/install_ollama_jetson.sh deleted file mode 100755 index 7e39a09..0000000 --- a/install_ollama_jetson.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env bash -# install_ollama_jetson.sh — add Jetson memory flags to the Ollama systemd unit -# -# WHY THIS SCRIPT EXISTS: -# `start_ollama.sh` only applies its OLLAMA_* env vars if you launch Ollama -# manually — but on the Jetson, Ollama is started by systemd at boot. Systemd -# ignores the shell script entirely, so none of the flags were ever reaching -# the live server. This installs a drop-in that systemd merges into the unit, -# so `systemctl restart ollama` picks up the flags. Confirmed by the log line -# `inference compute ... library=CUDA ... available="13.8 GiB"` — GPU works, -# but without OLLAMA_GPU_OVERHEAD Ollama claims all 13.8 GiB and the -# vision-encode pass OOMs the runner (seen as "llama runner process has -# terminated ... status code: 500" when an image is attached). -# -# Run once, from the Jetson: -# sudo ./install_ollama_jetson.sh -# -set -euo pipefail - -if [[ $EUID -ne 0 ]]; then - echo "This script must be run as root (it writes to /etc/systemd/system)." >&2 - echo "Re-run with: sudo $0" >&2 - exit 1 -fi - -DROPIN_DIR="/etc/systemd/system/ollama.service.d" -DROPIN_FILE="$DROPIN_DIR/marcus-jetson.conf" - -mkdir -p "$DROPIN_DIR" -cat > "$DROPIN_FILE" <<'EOF' -# Jetson memory caps for Marcus — do not edit live, re-run install_ollama_jetson.sh -[Service] -# Flash attention: ~30% less memory for attention tensors. -Environment="OLLAMA_FLASH_ATTENTION=1" -# Quantize KV cache to 8-bit (halves KV memory; negligible quality loss). -Environment="OLLAMA_KV_CACHE_TYPE=q8_0" -# Never hold two VL models simultaneously. -Environment="OLLAMA_MAX_LOADED_MODELS=1" -# CRITICAL: keep the model resident essentially forever. The previous 2m value -# meant that any pause in conversation longer than 2 min unloaded the model, -# and the NEXT "what do you see" paid another 60-90 s cold-load. That -# cold-load hammered unified memory + disk bandwidth hard enough to break -# Holosoma's 20 ms real-time deadline — inference times climbed from 2 ms to -# 153 ms and the robot fell. 24h means one cold-load per day (during -# warmup_vlm.sh while the robot is in squat), everything after is warm. -Environment="OLLAMA_KEEP_ALIVE=24h" -# Reserve 4 GiB of the 15 GiB iGPU for the rest of the system (YOLO + -# Holosoma + camera + Python heap + vision-encoder transient spike). -# Raised from 3 GiB after observing the robot wobble/fall during an -# `ask()` call — the vision encoder's peak activations were still eating -# into Holosoma's slack memory. -Environment="OLLAMA_GPU_OVERHEAD=4294967296" -# Cap the model's context window at the server level so Ollama's compute -# graph pre-allocation shrinks from 7.3 GiB (KvSize:2048 default) to -# ~3-4 GiB (KvSize:1024). Runtime num_ctx in request options is ignored -# for compute-graph sizing — only this env var affects the load-time -# allocation. 1024 tokens is enough for our prompts + short answers. -Environment="OLLAMA_CONTEXT_LENGTH=1024" -# Deprioritize Ollama so it never preempts Holosoma's real-time locomotion -# thread. IOSchedulingClass=idle = Ollama only gets disk bandwidth when -# nobody else wants it. Nice=10 = lowest normal priority. -IOSchedulingClass=idle -Nice=10 -# Make Ollama the preferred OOM-killer victim. If the 15 GiB Jetson iGPU + -# system memory fills up (e.g. swap thrashing after a vision request), -# kernel picks Ollama first — it auto-restarts, the model cold-loads on -# next request, Marcus and Holosoma keep running. Much better outcome than -# killing Marcus (robot brain) or Holosoma (balance control). -# Observed 2026-04-22: with oom_score_adj=0 across the board, the kernel -# killed Marcus's python3 (22 GB virtual, swapped) instead of Ollama -# (larger RSS but systemd-managed). This flips that. -OOMScoreAdjust=500 -EOF - -chmod 644 "$DROPIN_FILE" -echo "Wrote $DROPIN_FILE" -echo - -systemctl daemon-reload -systemctl restart ollama - -sleep 2 -echo "=== verification: these vars should now be in the live process ===" -journalctl -u ollama -n 40 --no-pager | grep -E "OLLAMA_FLASH_ATTENTION|OLLAMA_KV_CACHE_TYPE|OLLAMA_GPU_OVERHEAD|inference compute|vram" | tail -10 -echo -echo "If the 'inference compute' line shows library=CUDA and your flags appear" -echo "in the 'server config' dump above, you're done. Try 'what do you see' again." diff --git a/run.sh b/run.sh deleted file mode 100755 index 1337304..0000000 --- a/run.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env bash -# run.sh — one-command launcher for Marcus on the Jetson. -# -# What it does (safely, in order): -# 1. Ensures systemd Ollama is running (with drop-in flags). -# 2. Pulls the VLM model if it's missing. -# 3. Runs the VLM warmup (asks you to put the robot in squat first). -# 4. Health-gates iGPU placement and system memory. -# 5. Pauses for you to stand the robot up. -# 6. Launches Marcus. -# -# Fail-fast: any health check failure aborts BEFORE the robot stands. -# -# Usage: -# ./run.sh normal launch -# ./run.sh --skip-warmup skip warmup (only safe if already warm) -# ./run.sh --skip-stand-prompt don't pause for stand-up confirmation -# -# First time on a fresh Jetson (one-time, not run here): -# sudo ./install_ollama_jetson.sh - -set -e - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -cd "$SCRIPT_DIR" - -SKIP_WARMUP=0 -SKIP_STAND_PROMPT=0 -for arg in "$@"; do - case "$arg" in - --skip-warmup) SKIP_WARMUP=1 ;; - --skip-stand-prompt) SKIP_STAND_PROMPT=1 ;; - -h|--help) - grep '^#' "$0" | sed 's/^# \{0,1\}//' - exit 0 - ;; - *) - echo "Unknown argument: $arg" >&2 - echo "Use --help for usage." >&2 - exit 2 - ;; - esac -done - -# ── tiny color helpers ────────────────────────────────────────────── -BOLD='\033[1m'; GREEN='\033[32m'; YELLOW='\033[33m'; RED='\033[31m'; DIM='\033[2m'; OFF='\033[0m' -step() { echo -e "\n${BOLD}▶ $1${OFF}"; } -ok() { echo -e " ${GREEN}✓${OFF} $1"; } -warn() { echo -e " ${YELLOW}⚠${OFF} $1"; } -die() { echo -e " ${RED}✗${OFF} $1"; exit 1; } - -MODEL=$(python3 -c "import json; print(json.load(open('Config/config_Brain.json'))['ollama_model'])") -HOST=$(python3 -c "import json; print(json.load(open('Config/config_Brain.json'))['ollama_host'])") - -# ── 1. Ensure Ollama service is up ───────────────────────────────── -step "1/5 Ollama systemd service" -if ! systemctl is-active --quiet ollama; then - warn "service is not active — starting it" - sudo systemctl start ollama - sleep 2 -fi -if ! systemctl is-active --quiet ollama; then - die "failed to start Ollama; check: systemctl status ollama" -fi -ADJUST=$(systemctl show ollama -p OOMScoreAdjust --value) -if [[ "$ADJUST" != "500" ]]; then - warn "OOMScoreAdjust is '$ADJUST' (expected 500)" - warn "drop-in may not be installed — run: sudo ./install_ollama_jetson.sh" -fi -ok "service active (OOMScoreAdjust=$ADJUST)" - -# ── 2. Ensure the model is pulled ────────────────────────────────── -step "2/5 VLM model present" -if ! ollama list | awk 'NR>1 {print $1}' | grep -qx "$MODEL"; then - warn "$MODEL not in store — pulling (~2.2 GB, one-time)" - ollama pull "$MODEL" -fi -ok "$MODEL available in Ollama store" - -# ── 3. Warmup (robot-in-squat safety banner lives inside the script) ── -step "3/5 VLM warmup" -if [[ $SKIP_WARMUP -eq 1 ]]; then - warn "skipped by --skip-warmup" -else - ./warmup_vlm.sh "$MODEL" -fi - -# ── 4. Health gates ──────────────────────────────────────────────── -step "4/5 Health check" - -# 4a. PROCESSOR placement -PS_LINE=$(ollama ps | awk -v m="$MODEL" '$1==m {for(i=1;i<=NF;i++) if($i ~ /GPU/) {print $(i-1), $i; exit}}') -PROCESSOR=$(ollama ps | awk -v m="$MODEL" 'NR>1 && $1==m { - for(i=4;i<=NF;i++) if($i=="GPU" || $i=="CPU/GPU" || $i=="CPU") {print $(i-1) " " $i; break} -}') -if [[ -z "$PROCESSOR" ]]; then - warn "model not loaded — running without prewarm; first vision call will cold-load" -elif echo "$PROCESSOR" | grep -qE "^100% GPU"; then - ok "placement: $PROCESSOR (ideal)" -elif echo "$PROCESSOR" | grep -qE "CPU/GPU"; then - SPLIT=$(echo "$PROCESSOR" | awk '{print $1}') - warn "placement: $PROCESSOR — partial CPU offload will slow vision queries" - warn "if unacceptable, lower OLLAMA_GPU_OVERHEAD in install_ollama_jetson.sh" -else - warn "placement: $PROCESSOR" -fi - -# 4b. system memory -read -r MEM_USED MEM_AVAIL SWAP_USED <<< "$(free -m | awk ' - /^Mem:/ { used=$3; avail=$7 } - /^Swap:/ { sused=$3 } - END { print used, avail, sused } -')" -echo " ${DIM}mem used: ${MEM_USED} MiB available: ${MEM_AVAIL} MiB swap used: ${SWAP_USED} MiB${OFF}" -if (( SWAP_USED > 500 )); then - die "swap in use (${SWAP_USED} MiB) — memory is already tight, would thrash under load" -fi -if (( MEM_AVAIL < 1000 )); then - die "only ${MEM_AVAIL} MiB available — Holosoma/Marcus/camera won't have room" -fi -ok "memory healthy" - -# ── 5. Stand the robot, then launch ──────────────────────────────── -step "5/5 Launch Marcus" -if [[ $SKIP_STAND_PROMPT -eq 0 ]]; then - echo -e " ${YELLOW}Robot can now stand up.${OFF}" - echo -e " ${DIM}Press ENTER when the robot is standing, or Ctrl-C to abort.${OFF}" - read -r -fi -echo - -exec python3 run_marcus.py diff --git a/start_ollama.sh b/start_ollama.sh deleted file mode 100755 index 4de9f50..0000000 --- a/start_ollama.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -# start_ollama.sh — DEPRECATED, do not use. -# -# This script launches Ollama as the current (unitree) user and completely -# BYPASSES the systemd drop-in at /etc/systemd/system/ollama.service.d/, -# which is where all the safety-critical flags live: -# -# OOMScoreAdjust=500 (makes Ollama the preferred OOM victim) -# IOSchedulingClass=idle (so disk reads don't starve Holosoma) -# Nice=10 (lowest normal scheduler priority) -# OLLAMA_GPU_OVERHEAD=4 GiB (reserves iGPU for YOLO/Holosoma/camera) -# OLLAMA_KEEP_ALIVE=24h (model stays resident all day) -# OLLAMA_CONTEXT_LENGTH=1024 (shrinks compute graph) -# OLLAMA_FLASH_ATTENTION=1 -# OLLAMA_KV_CACHE_TYPE=q8_0 -# OLLAMA_MAX_LOADED_MODELS=1 -# -# Running this script instead of systemd means NONE of those protections -# are active, and the robot WILL fall the next time a vision query runs. -# -# CORRECT WAY: -# sudo systemctl start ollama # uses the drop-in -# ./warmup_vlm.sh # then warm up -# python3 run_marcus.py # then start Marcus -# -# ═════════════════════════════════════════════════════════════════════ -# REFUSING TO RUN. Delete this block only if you know what you're doing. -# ═════════════════════════════════════════════════════════════════════ -echo "start_ollama.sh is DEPRECATED. Use: sudo systemctl start ollama" >&2 -echo " (see comments at the top of this file for why)" >&2 -exit 1 - -pkill -f "ollama (runner|serve)" 2>/dev/null -sleep 1 - -export OLLAMA_FLASH_ATTENTION=1 -export OLLAMA_KV_CACHE_TYPE=q8_0 -export OLLAMA_KEEP_ALIVE=2m -export OLLAMA_MAX_LOADED_MODELS=1 -# Reserve 2 GiB of the Jetson's 15 GiB iGPU for the rest of the system -# (YOLO in CUDA FP16, Holosoma, camera, Python heap). Without this, Ollama -# assumes the full 13.8 GiB "available" is its to use and sizes its compute -# graph that way — which works for text, but the vision-encode pass of -# Qwen2.5-VL then pushes total allocation past physical memory and the -# runner dies with status 500. -export OLLAMA_GPU_OVERHEAD=2147483648 - -if [[ "$1" == "--fg" ]]; then - echo "Running ollama in foreground..." - ollama serve -else - ollama serve > /tmp/ollama.log 2>&1 & - sleep 3 - if curl -sf http://localhost:11434/api/version > /dev/null; then - echo "✓ Ollama started (pid $(pgrep -f 'ollama serve'))" - echo " logs: tail -f /tmp/ollama.log" - echo " stop: pkill -f 'ollama serve'" - else - echo "✗ Ollama failed to start — see /tmp/ollama.log" - exit 1 - fi -fi diff --git a/warmup_vlm.sh b/warmup_vlm.sh deleted file mode 100755 index dbd5b48..0000000 --- a/warmup_vlm.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash -# warmup_vlm.sh — pre-load Qwen2.5-VL into iGPU BEFORE the robot stands up. -# -# WHY THIS EXISTS: -# Cold-loading a 3 GB vision model on the Jetson's 16 GB unified memory -# hammers disk + memory bandwidth hard enough to blow Holosoma's 20 ms -# real-time deadline. If that happens while the robot is standing, Holosoma -# loses balance control and the robot falls (observed 2026-04-22 — -# RL FPS crashed from 56 → 10 during cold-load, OOM killer reaped Holosoma). -# -# SAFE OPERATING PROCEDURE (do this once per boot): -# 1. Boot robot. Keep it in squat / damping mode (NOT standing). -# 2. Run: ./warmup_vlm.sh -# 3. Wait for "Warmup complete" (~60-90 s). -# 4. NOW raise the robot to standing. -# 5. Run: python3 run_marcus.py -# -# After warmup, OLLAMA_KEEP_ALIVE=24h keeps the model hot for the rest of the -# day, so every "what do you see" is fast (~2-5 s) and Holosoma stays -# healthy. Never cold-load a vision model while the robot is standing. - -set -e - -MODEL="${1:-qwen2.5vl:3b}" -HOST="http://127.0.0.1:11434" - -# Pull num_batch / num_ctx from config_Brain.json. CRITICAL: these must match -# what Marcus will send at runtime, otherwise Ollama evicts the cached runner -# on the FIRST real vision command and re-instantiates it — the overlap of -# old-runner-unloading + new-runner-loading triggers an iGPU OOM and the -# runner dies with status 500. Symptom: warmup succeeds, first `what do you -# see` crashes 10-30 s later with "llama runner process has terminated". -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -NUM_BATCH=$(python3 -c "import json; print(json.load(open('$SCRIPT_DIR/Config/config_Brain.json'))['num_batch'])" 2>/dev/null || echo 64) -NUM_CTX=$(python3 -c "import json; print(json.load(open('$SCRIPT_DIR/Config/config_Brain.json'))['num_ctx'])" 2>/dev/null || echo 1024) - -# Sanity: is ollama reachable? -if ! curl -sf "$HOST/api/version" > /dev/null; then - echo "✗ Ollama is not running on $HOST" - echo " Start it: sudo systemctl start ollama" - exit 1 -fi - -# Sanity: is the model in the store? -if ! ollama list | awk 'NR>1 {print $1}' | grep -qx "$MODEL"; then - echo "✗ Model '$MODEL' not found in Ollama store" - echo " Pull it: ollama pull $MODEL" - exit 1 -fi - -# Is the model ALREADY resident? `ollama ps` lists running models with their -# keep-alive. If qwen2.5vl:3b is there, a ping is ~1 s and the robot doesn't -# need to squat for it. Only ask the user to squat if we're about to do the -# real 60-90 s disk + iGPU hammer. -ALREADY_WARM=0 -if ollama ps 2>/dev/null | awk 'NR>1 {print $1}' | grep -qx "$MODEL"; then - ALREADY_WARM=1 -fi - -if [[ $ALREADY_WARM -eq 1 ]]; then - echo "→ $MODEL is already resident (keep-alive). Pinging to confirm..." -else - cat <<'BANNER' -════════════════════════════════════════════════════════════════════ - VLM WARMUP — put the robot in SQUAT / DAMPING mode first! -════════════════════════════════════════════════════════════════════ - - The next 60-90 s will hammer disk + memory bandwidth. Holosoma - cannot balance through this. If the robot is standing, IT WILL FALL. - - Press ENTER when the robot is safely in squat / damping. - Press Ctrl-C to abort. - -BANNER - read -r - echo "→ Cold-loading $MODEL into iGPU (60-90 s, do not interrupt)..." - echo " num_batch=$NUM_BATCH num_ctx=$NUM_CTX (matching Marcus's runtime config)" - echo " Watching tegrastats in a second window is useful — GPU should spike." - echo -fi - -START=$(date +%s) - -# Send a tiny text-only request. Ollama loads the model on first request; -# we don't need vision here — just getting weights resident is what takes the -# long time. A text-only warmup also avoids needing a camera frame. -# -# --keepalive 24h matches the systemd config and prevents the server from -# unloading the model after the default 5 min. -RESPONSE=$(curl -s "$HOST/api/generate" \ - -H 'Content-Type: application/json' \ - -d "{\"model\":\"$MODEL\",\"prompt\":\"ok\",\"stream\":false,\"keep_alive\":\"24h\",\"options\":{\"num_predict\":1,\"num_batch\":$NUM_BATCH,\"num_ctx\":$NUM_CTX}}") - -END=$(date +%s) -ELAPSED=$((END - START)) - -if echo "$RESPONSE" | grep -q '"response"'; then - if [[ $ALREADY_WARM -eq 1 ]]; then - echo "✓ Model already warm — no cold-load needed (${ELAPSED}s ping)" - else - echo "✓ Warmup complete in ${ELAPSED}s — model is resident for 24h" - echo " You can now stand the robot and run: python3 run_marcus.py" - fi -else - echo "✗ Warmup failed after ${ELAPSED}s" - echo " Response: $RESPONSE" - echo " Check: journalctl -u ollama --since '2 minutes ago' --no-pager | tail -40" - exit 1 -fi