all detectors
This commit is contained in:
@@ -0,0 +1 @@
|
||||
--agent-import-path is deprecated; use --agent instead.
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"job_name": "regrade-1-reward-0.3000-wNYgXoP",
|
||||
"jobs_dir": "harbor-jobs",
|
||||
"environment": {
|
||||
"type": "docker",
|
||||
"delete": false
|
||||
},
|
||||
"verifier": {
|
||||
"env": {
|
||||
"ANTHROPIC_CUSTOM_HEADERS": "X-Surge-Client-Metadata: {\"origin\":\"harbor-grading\"}"
|
||||
}
|
||||
},
|
||||
"agents": [
|
||||
{
|
||||
"import_path": "replay_agent:ReplayAgent",
|
||||
"kwargs": {
|
||||
"reference_run_dir": "/home/eric/workspaces/dataannotation/current-project/worker-toolkit-potion-polyglot/harbor-tasks/mishandled_pro_v2/reference-runs/reward-0.3000-wNYgXoP",
|
||||
"source_agent_import_path": "codex_agent:SystemNodeCodex",
|
||||
"source_model_name": "gpt-5.6-sol"
|
||||
}
|
||||
}
|
||||
],
|
||||
"tasks": [
|
||||
{
|
||||
"path": "/home/eric/workspaces/dataannotation/current-project/worker-toolkit-potion-polyglot/harbor-tasks/mishandled_pro_v2"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
Skipping image OS validation for hb__3b6772e9c502dcc9691720743040aab6: docker inspect returned 1
|
||||
Collecting main service artifacts
|
||||
The verifier.env contains an API key (often the case for LLM-based verifiers). You will incur costs associated with the API calls.
|
||||
Trial mishandled_pro_v2__8qttC5x cancelled
|
||||
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"schema_version": 2,
|
||||
"created_at": "2026-09-29T23:08:21.784339Z",
|
||||
"harbor": {
|
||||
"version": "0.20.0",
|
||||
"is_editable": false
|
||||
},
|
||||
"n_concurrent_trials": 4,
|
||||
"retry": {
|
||||
"max_retries": 0,
|
||||
"exclude_exceptions": [
|
||||
"ModelNotFoundError",
|
||||
"VerifierOutputParseError",
|
||||
"AgentAuthenticationError",
|
||||
"AgentTimeoutError",
|
||||
"VerifierTimeoutError",
|
||||
"RewardFileEmptyError",
|
||||
"RewardFileNotFoundError",
|
||||
"AgentSafetyRefusalError",
|
||||
"ApiUsageLimitError"
|
||||
],
|
||||
"wait_multiplier": 1.0,
|
||||
"min_wait_sec": 1.0,
|
||||
"max_wait_sec": 60.0
|
||||
},
|
||||
"trials": [
|
||||
{
|
||||
"schema_version": 1,
|
||||
"task": {
|
||||
"name": "mishandled_pro_v2",
|
||||
"type": "local",
|
||||
"digest": "sha256:aa5dd26e638654b00dca5f57d272908ed248aac5c789af1755c6665ab0519f09",
|
||||
"path": "/home/eric/workspaces/dataannotation/current-project/worker-toolkit-potion-polyglot/harbor-tasks/mishandled_pro_v2"
|
||||
},
|
||||
"install_only": false,
|
||||
"timeout_multiplier": 1.0,
|
||||
"agent": {
|
||||
"import_path": "replay_agent:ReplayAgent",
|
||||
"skills": [],
|
||||
"resume_trajectory": false,
|
||||
"extra_allowed_hosts": [],
|
||||
"kwargs": {
|
||||
"reference_run_dir": "/home/eric/workspaces/dataannotation/current-project/worker-toolkit-potion-polyglot/harbor-tasks/mishandled_pro_v2/reference-runs/reward-0.3000-wNYgXoP",
|
||||
"source_agent_import_path": "codex_agent:SystemNodeCodex",
|
||||
"source_model_name": "gpt-5.6-sol"
|
||||
},
|
||||
"mcp_servers": []
|
||||
},
|
||||
"skills": [],
|
||||
"environment": {
|
||||
"type": "docker",
|
||||
"force_build": false,
|
||||
"delete": false,
|
||||
"cpu_enforcement_policy": "auto",
|
||||
"memory_enforcement_policy": "auto",
|
||||
"extra_docker_compose": [],
|
||||
"kwargs": {},
|
||||
"extra_allowed_hosts": []
|
||||
},
|
||||
"verifier": {
|
||||
"env": {
|
||||
"ANTHROPIC_CUSTOM_HEADERS": "X-Surge-Client-Metadata: {\"origin\":\"harbor-grading\"}"
|
||||
},
|
||||
"disable": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"source": "/logs/artifacts",
|
||||
"destination": "artifacts/logs/artifacts",
|
||||
"type": "directory",
|
||||
"status": "empty",
|
||||
"service": null
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"task": {
|
||||
"path": "/home/eric/workspaces/dataannotation/current-project/worker-toolkit-potion-polyglot/harbor-tasks/mishandled_pro_v2"
|
||||
},
|
||||
"trial_name": "mishandled_pro_v2__8qttC5x",
|
||||
"trials_dir": "harbor-jobs/regrade-1-reward-0.3000-wNYgXoP",
|
||||
"agent": {
|
||||
"import_path": "replay_agent:ReplayAgent",
|
||||
"kwargs": {
|
||||
"reference_run_dir": "/home/eric/workspaces/dataannotation/current-project/worker-toolkit-potion-polyglot/harbor-tasks/mishandled_pro_v2/reference-runs/reward-0.3000-wNYgXoP",
|
||||
"source_agent_import_path": "codex_agent:SystemNodeCodex",
|
||||
"source_model_name": "gpt-5.6-sol"
|
||||
}
|
||||
},
|
||||
"environment": {
|
||||
"type": "docker",
|
||||
"delete": false
|
||||
},
|
||||
"verifier": {
|
||||
"env": {
|
||||
"ANTHROPIC_CUSTOM_HEADERS": "X-Surge-Client-Metadata: {\"origin\":\"harbor-grading\"}"
|
||||
}
|
||||
},
|
||||
"job_id": "18327e53-0e9a-4a10-92ec-43208f719152"
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
Traceback (most recent call last):
|
||||
File "/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 195, in run
|
||||
return runner.run(main)
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 118, in run
|
||||
return self._loop.run_until_complete(task)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
|
||||
self.run_forever()
|
||||
File "/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
|
||||
self._run_once()
|
||||
File "/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 1961, in _run_once
|
||||
event_list = self._selector.select(timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/selectors.py", line 468, in select
|
||||
fd_event_list = self._selector.poll(timeout, max_ev)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/cli/jobs.py", line 317, in _handle_sigterm
|
||||
raise KeyboardInterrupt
|
||||
KeyboardInterrupt
|
||||
|
||||
During handling of the above exception, another exception occurred:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/trial/trial.py", line 354, in run
|
||||
await self._run()
|
||||
File "/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/trial/single_step.py", line 52, in _run
|
||||
await self._run_verifier()
|
||||
File "/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/trial/single_step.py", line 105, in _run_verifier
|
||||
self.result.verifier_result = await self._run_shared_verifier(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/trial/trial.py", line 535, in _run_shared_verifier
|
||||
return await asyncio.wait_for(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 520, in wait_for
|
||||
return await fut
|
||||
^^^^^^^^^
|
||||
File "/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/verifier/verifier.py", line 199, in verify
|
||||
await self.environment.exec(
|
||||
File "/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1096, in exec
|
||||
return await self._compose_exec(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1173, in _compose_exec
|
||||
return await self._run_docker_compose_command(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 649, in _run_docker_compose_command
|
||||
result = await self._collect_buffered_output(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 679, in _collect_buffered_output
|
||||
stdout_bytes, stderr_bytes = await process.communicate(input=stdin_data)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 201, in communicate
|
||||
stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 181, in _read_stream
|
||||
output = await stream.read()
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 706, in read
|
||||
block = await self.read(self._limit)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 713, in read
|
||||
await self._wait_for_data('read')
|
||||
File "/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 545, in _wait_for_data
|
||||
await self._waiter
|
||||
asyncio.exceptions.CancelledError
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"task": {
|
||||
"name": "mishandled_pro_v2",
|
||||
"type": "local",
|
||||
"digest": "sha256:aa5dd26e638654b00dca5f57d272908ed248aac5c789af1755c6665ab0519f09",
|
||||
"path": "/home/eric/workspaces/dataannotation/current-project/worker-toolkit-potion-polyglot/harbor-tasks/mishandled_pro_v2"
|
||||
},
|
||||
"install_only": false,
|
||||
"timeout_multiplier": 1.0,
|
||||
"agent": {
|
||||
"import_path": "replay_agent:ReplayAgent",
|
||||
"skills": [],
|
||||
"resume_trajectory": false,
|
||||
"extra_allowed_hosts": [],
|
||||
"kwargs": {
|
||||
"reference_run_dir": "/home/eric/workspaces/dataannotation/current-project/worker-toolkit-potion-polyglot/harbor-tasks/mishandled_pro_v2/reference-runs/reward-0.3000-wNYgXoP",
|
||||
"source_agent_import_path": "codex_agent:SystemNodeCodex",
|
||||
"source_model_name": "gpt-5.6-sol"
|
||||
},
|
||||
"mcp_servers": []
|
||||
},
|
||||
"skills": [],
|
||||
"environment": {
|
||||
"type": "docker",
|
||||
"force_build": false,
|
||||
"delete": false,
|
||||
"cpu_enforcement_policy": "auto",
|
||||
"memory_enforcement_policy": "auto",
|
||||
"extra_docker_compose": [],
|
||||
"kwargs": {},
|
||||
"extra_allowed_hosts": []
|
||||
},
|
||||
"verifier": {
|
||||
"env": {
|
||||
"ANTHROPIC_CUSTOM_HEADERS": "X-Surge-Client-Metadata: {\"origin\":\"harbor-grading\"}"
|
||||
},
|
||||
"disable": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
{
|
||||
"id": "63dc902b-1f39-47ed-aef0-90b547814297",
|
||||
"task_name": "mishandled_pro_v2",
|
||||
"trial_name": "mishandled_pro_v2__8qttC5x",
|
||||
"trial_uri": "file:///home/eric/workspaces/dataannotation/current-project/worker-toolkit-potion-polyglot/harbor-jobs/regrade-1-reward-0.3000-wNYgXoP/mishandled_pro_v2__8qttC5x",
|
||||
"task_id": {
|
||||
"path": "/home/eric/workspaces/dataannotation/current-project/worker-toolkit-potion-polyglot/harbor-tasks/mishandled_pro_v2"
|
||||
},
|
||||
"source": null,
|
||||
"task_checksum": "0fcaf8025b587147f2f03d7ce6702572765a8d92e6817dcb57f153b10fedf94c",
|
||||
"config": {
|
||||
"task": {
|
||||
"path": "/home/eric/workspaces/dataannotation/current-project/worker-toolkit-potion-polyglot/harbor-tasks/mishandled_pro_v2",
|
||||
"git_url": null,
|
||||
"git_commit_id": null,
|
||||
"name": null,
|
||||
"ref": null,
|
||||
"overwrite": false,
|
||||
"download_dir": null,
|
||||
"source": null
|
||||
},
|
||||
"trial_name": "mishandled_pro_v2__8qttC5x",
|
||||
"trials_dir": "harbor-jobs/regrade-1-reward-0.3000-wNYgXoP",
|
||||
"install_only": false,
|
||||
"timeout_multiplier": 1.0,
|
||||
"agent_timeout_multiplier": null,
|
||||
"verifier_timeout_multiplier": null,
|
||||
"agent_setup_timeout_multiplier": null,
|
||||
"environment_build_timeout_multiplier": null,
|
||||
"agent": {
|
||||
"name": null,
|
||||
"import_path": "replay_agent:ReplayAgent",
|
||||
"model_name": null,
|
||||
"n_concurrent": null,
|
||||
"concurrency_group": null,
|
||||
"skills": [],
|
||||
"override_timeout_sec": null,
|
||||
"override_setup_timeout_sec": null,
|
||||
"max_timeout_sec": null,
|
||||
"resume_trajectory": false,
|
||||
"load_trajectory": null,
|
||||
"extra_allowed_hosts": [],
|
||||
"kwargs": {
|
||||
"reference_run_dir": "/home/eric/workspaces/dataannotation/current-project/worker-toolkit-potion-polyglot/harbor-tasks/mishandled_pro_v2/reference-runs/reward-0.3000-wNYgXoP",
|
||||
"source_agent_import_path": "codex_agent:SystemNodeCodex",
|
||||
"source_model_name": "gpt-5.6-sol"
|
||||
},
|
||||
"mcp_servers": []
|
||||
},
|
||||
"environment": {
|
||||
"type": "docker",
|
||||
"import_path": null,
|
||||
"force_build": false,
|
||||
"delete": false,
|
||||
"cpu_enforcement_policy": "auto",
|
||||
"memory_enforcement_policy": "auto",
|
||||
"override_cpus": null,
|
||||
"override_memory_mb": null,
|
||||
"override_storage_mb": null,
|
||||
"override_gpus": null,
|
||||
"override_tpu": null,
|
||||
"mounts": null,
|
||||
"extra_docker_compose": [],
|
||||
"kwargs": {},
|
||||
"extra_allowed_hosts": []
|
||||
},
|
||||
"verifier": {
|
||||
"override_timeout_sec": null,
|
||||
"max_timeout_sec": null,
|
||||
"env": {
|
||||
"ANTHROPIC_CUSTOM_HEADERS": "X-Surge-Client-Metadata: {\"origin\":\"harbor-grading\"}"
|
||||
},
|
||||
"disable": false
|
||||
},
|
||||
"artifacts": [],
|
||||
"extra_instruction_paths": [],
|
||||
"job_id": "18327e53-0e9a-4a10-92ec-43208f719152"
|
||||
},
|
||||
"agent_info": {
|
||||
"name": "replay",
|
||||
"version": "1.0.0",
|
||||
"model_info": null
|
||||
},
|
||||
"agent_result": {
|
||||
"n_input_tokens": null,
|
||||
"n_cache_tokens": null,
|
||||
"n_output_tokens": null,
|
||||
"cost_usd": null,
|
||||
"rollout_details": null,
|
||||
"metadata": null
|
||||
},
|
||||
"verifier_result": null,
|
||||
"exception_info": {
|
||||
"exception_type": "CancelledError",
|
||||
"exception_message": "",
|
||||
"exception_traceback": "Traceback (most recent call last):\n File \"/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py\", line 195, in run\n return runner.run(main)\n ^^^^^^^^^^^^^^^^\n File \"/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py\", line 118, in run\n return self._loop.run_until_complete(task)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py\", line 678, in run_until_complete\n self.run_forever()\n File \"/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py\", line 645, in run_forever\n self._run_once()\n File \"/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py\", line 1961, in _run_once\n event_list = self._selector.select(timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/selectors.py\", line 468, in select\n fd_event_list = self._selector.poll(timeout, max_ev)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/cli/jobs.py\", line 317, in _handle_sigterm\n raise KeyboardInterrupt\nKeyboardInterrupt\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/trial/trial.py\", line 354, in run\n await self._run()\n File \"/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/trial/single_step.py\", line 52, in _run\n await self._run_verifier()\n File \"/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/trial/single_step.py\", line 105, in _run_verifier\n self.result.verifier_result = await self._run_shared_verifier(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/trial/trial.py\", line 535, in _run_shared_verifier\n return await asyncio.wait_for(\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py\", line 520, in wait_for\n return await fut\n ^^^^^^^^^\n File \"/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/verifier/verifier.py\", line 199, in verify\n await self.environment.exec(\n File \"/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/environments/docker/docker.py\", line 1096, in exec\n return await self._compose_exec(\n ^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/environments/docker/docker.py\", line 1173, in _compose_exec\n return await self._run_docker_compose_command(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/environments/docker/docker.py\", line 649, in _run_docker_compose_command\n result = await self._collect_buffered_output(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/.local/share/uv/tools/harbor/lib/python3.12/site-packages/harbor/environments/docker/docker.py\", line 679, in _collect_buffered_output\n stdout_bytes, stderr_bytes = await process.communicate(input=stdin_data)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py\", line 201, in communicate\n stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py\", line 181, in _read_stream\n output = await stream.read()\n ^^^^^^^^^^^^^^^^^^^\n File \"/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py\", line 706, in read\n block = await self.read(self._limit)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py\", line 713, in read\n await self._wait_for_data('read')\n File \"/root/.local/share/uv/python/cpython-3.12.14-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py\", line 545, in _wait_for_data\n await self._waiter\nasyncio.exceptions.CancelledError\n",
|
||||
"occurred_at": "2026-09-29T23:10:37.289126"
|
||||
},
|
||||
"started_at": "2026-09-29T23:08:22.105945Z",
|
||||
"finished_at": "2026-09-29T23:10:41.542725Z",
|
||||
"environment_setup": {
|
||||
"started_at": "2026-09-29T23:08:22.291556Z",
|
||||
"finished_at": "2026-09-29T23:08:28.805700Z"
|
||||
},
|
||||
"agent_setup": {
|
||||
"started_at": "2026-09-29T23:08:28.805790Z",
|
||||
"finished_at": "2026-09-29T23:08:28.805888Z"
|
||||
},
|
||||
"agent_execution": {
|
||||
"started_at": "2026-09-29T23:08:28.806034Z",
|
||||
"finished_at": "2026-09-29T23:08:29.155855Z"
|
||||
},
|
||||
"verifier": {
|
||||
"started_at": "2026-09-29T23:08:29.641423Z",
|
||||
"finished_at": "2026-09-29T23:10:37.282062Z"
|
||||
},
|
||||
"step_results": null
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
Skipping image OS validation for hb__3b6772e9c502dcc9691720743040aab6: docker inspect returned 1
|
||||
Collecting main service artifacts
|
||||
The verifier.env contains an API key (often the case for LLM-based verifiers). You will incur costs associated with the API calls.
|
||||
Trial mishandled_pro_v2__8qttC5x cancelled
|
||||
@@ -0,0 +1,53 @@
|
||||
const mongoose = require('mongoose')
|
||||
const Schema = mongoose.Schema
|
||||
|
||||
const VoiceCloningSchema = Schema(
|
||||
{
|
||||
userId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: 'User',
|
||||
required: true,
|
||||
},
|
||||
userAudioProfileId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: 'UserAudioProfile',
|
||||
required: true,
|
||||
},
|
||||
status: {
|
||||
type: String,
|
||||
required: false,
|
||||
default: 'created',
|
||||
},
|
||||
tier: {
|
||||
type: String,
|
||||
required: false,
|
||||
trim: true,
|
||||
lowercase: true,
|
||||
default: null,
|
||||
},
|
||||
input: {
|
||||
type: Schema.Types.Mixed,
|
||||
default: null,
|
||||
},
|
||||
training_model: {
|
||||
type: Schema.Types.Mixed,
|
||||
default: null,
|
||||
},
|
||||
metadata: {
|
||||
type: Schema.Types.Mixed,
|
||||
default: null,
|
||||
},
|
||||
deleted: {
|
||||
type: Boolean,
|
||||
required: true,
|
||||
default: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
timestamps: true,
|
||||
}
|
||||
)
|
||||
|
||||
module.exports =
|
||||
mongoose.models.VoiceCloning ||
|
||||
mongoose.model('VoiceCloning', VoiceCloningSchema)
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "potion-voice",
|
||||
"version": "1.0.0",
|
||||
"description": "This will handle the voice cloning jobs",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "node test/job_contract.test.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@bugsnag/js": "^7.3.5",
|
||||
"aws-sdk": "^2.752.0",
|
||||
"fs-extra": "^9.0.1",
|
||||
"mongoose": "^6.8.0",
|
||||
"pm2": "^5.2.0",
|
||||
"rimraf": "^3.0.2",
|
||||
"uuid": "^8.3.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"aws-code-deploy": "^1.0.11"
|
||||
},
|
||||
"author": "potion Team",
|
||||
"license": "ISC"
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
'use strict'
|
||||
|
||||
const assert = require('assert')
|
||||
const {
|
||||
DEFAULT_TIER,
|
||||
PRO_V2_TIER,
|
||||
normalizeTier,
|
||||
parseJobEnvelope,
|
||||
resolveTierConfig,
|
||||
} = require('../voice-cloning-job-handler/voice_cloning/job_contract')
|
||||
|
||||
const tests = []
|
||||
const test = (name, run) => tests.push({ name, run })
|
||||
|
||||
test('parses the legacy Mongoose queue envelope', () => {
|
||||
const parsed = parseJobEnvelope({
|
||||
_doc: {
|
||||
_id: 'clone-1',
|
||||
userAudioProfileId: 'profile-1',
|
||||
input: [],
|
||||
metadata: { directoryName: 'voice-1' },
|
||||
},
|
||||
env: 'staging',
|
||||
})
|
||||
|
||||
assert.strictEqual(parsed._id, 'clone-1')
|
||||
assert.strictEqual(parsed.userAudioProfileId, 'profile-1')
|
||||
assert.strictEqual(parsed.env, 'staging')
|
||||
assert.strictEqual(parsed.tier, DEFAULT_TIER)
|
||||
})
|
||||
|
||||
test('parses a plain pro_v2 queue job', () => {
|
||||
const parsed = parseJobEnvelope({
|
||||
id: 'clone-2',
|
||||
user_audio_profile_id: 'profile-2',
|
||||
tier: 'pro_v2',
|
||||
environment: 'production',
|
||||
input: [],
|
||||
metadata: { directoryName: 'voice-2' },
|
||||
})
|
||||
|
||||
assert.strictEqual(parsed._id, 'clone-2')
|
||||
assert.strictEqual(parsed.userAudioProfileId, 'profile-2')
|
||||
assert.strictEqual(parsed.env, 'production')
|
||||
assert.strictEqual(parsed.tier, PRO_V2_TIER)
|
||||
})
|
||||
|
||||
test('parses a nested job and reads its tier from metadata', () => {
|
||||
const parsed = parseJobEnvelope({
|
||||
env: 'staging',
|
||||
job: {
|
||||
_id: 'clone-3',
|
||||
userAudioProfileId: 'profile-3',
|
||||
metadata: { directoryName: 'voice-3', tier: 'PRO_V2' },
|
||||
},
|
||||
})
|
||||
|
||||
assert.strictEqual(parsed._id, 'clone-3')
|
||||
assert.strictEqual(parsed.env, 'staging')
|
||||
assert.strictEqual(parsed.tier, PRO_V2_TIER)
|
||||
})
|
||||
|
||||
test('uses the pro_v2 training configuration', () => {
|
||||
const config = resolveTierConfig(' PRO_V2 ', {
|
||||
PRO_V2_DATASET_PRESET: 'pro-dataset',
|
||||
PRO_V2_BASELINE_MODEL_PATH: '/models/pro-v2.pth',
|
||||
PRO_V2_CHECKPOINT_NAME: 'best_model.pth',
|
||||
})
|
||||
|
||||
assert.deepStrictEqual(config, {
|
||||
tier: PRO_V2_TIER,
|
||||
datasetPreset: 'pro-dataset',
|
||||
baselineModelPath: '/models/pro-v2.pth',
|
||||
checkpointName: 'best_model.pth',
|
||||
})
|
||||
})
|
||||
|
||||
test('pro_v2 falls back to the deployed v2 model assets', () => {
|
||||
assert.deepStrictEqual(resolveTierConfig('pro_v2', {}), {
|
||||
tier: PRO_V2_TIER,
|
||||
datasetPreset: 'potion_voice_cloning',
|
||||
baselineModelPath:
|
||||
'../voice-cloning/pretrained-models/checkpoint_365000.pth',
|
||||
checkpointName: 'checkpoint_365200.pth',
|
||||
})
|
||||
})
|
||||
|
||||
test('rejects invalid non-string tiers', () => {
|
||||
assert.throws(() => normalizeTier({ name: 'pro_v2' }), /must be a string/)
|
||||
})
|
||||
|
||||
test('persists a normalized pro_v2 tier on cloning jobs', () => {
|
||||
const mongoose = require('mongoose')
|
||||
const VoiceCloning = require('../app/services/voice_cloning/voice_cloning_model')
|
||||
const cloning = new VoiceCloning({
|
||||
userId: new mongoose.Types.ObjectId(),
|
||||
userAudioProfileId: new mongoose.Types.ObjectId(),
|
||||
tier: ' PRO_V2 ',
|
||||
})
|
||||
|
||||
assert.strictEqual(cloning.tier, PRO_V2_TIER)
|
||||
assert.strictEqual(cloning.status, 'created')
|
||||
})
|
||||
|
||||
test('does not accept a null cloning-job state update', async () => {
|
||||
const voiceCloningService = require('../voice-cloning-job-handler/voice_cloning')
|
||||
const originalUpdate = voiceCloningService.update
|
||||
voiceCloningService.update = async () => null
|
||||
|
||||
try {
|
||||
const { updateVoiceCloning } = require('../voice-cloning-job-handler')
|
||||
await assert.rejects(
|
||||
updateVoiceCloning({ _id: 'missing', status: 'processing' }),
|
||||
/was not found/
|
||||
)
|
||||
} finally {
|
||||
voiceCloningService.update = originalUpdate
|
||||
}
|
||||
})
|
||||
|
||||
const runTests = async () => {
|
||||
let failures = 0
|
||||
for (const { name, run } of tests) {
|
||||
try {
|
||||
await run()
|
||||
console.log(`ok - ${name}`)
|
||||
} catch (error) {
|
||||
failures += 1
|
||||
console.error(`not ok - ${name}`)
|
||||
console.error(error.stack || error)
|
||||
}
|
||||
}
|
||||
|
||||
if (failures) process.exitCode = 1
|
||||
}
|
||||
|
||||
runTests()
|
||||
@@ -0,0 +1,399 @@
|
||||
const fs = require('fs')
|
||||
const https = require('https')
|
||||
const exec = require('child_process').exec
|
||||
const AWS = require('aws-sdk')
|
||||
|
||||
const Bugsnag = require('@bugsnag/js')
|
||||
const mongoose = require('mongoose')
|
||||
const version = require('./package.json').version
|
||||
const sqs = require('../app/services/sqs')
|
||||
const s3 = require('../app/services/s3')
|
||||
const voiceCloningService = require('./voice_cloning')
|
||||
const userAudioProfileService = require('./user_audio_profile')
|
||||
const {
|
||||
parseJobEnvelope,
|
||||
resolveTierConfig,
|
||||
} = require('./voice_cloning/job_contract')
|
||||
|
||||
AWS.config.update({ region: 'us-west-2' })
|
||||
const sqsQueueUrl = process.env.SQS_URL
|
||||
const mongoUriDev = process.env.MONGODB_URI_DEV
|
||||
const mongoUriStaging = process.env.MONGODB_URI_STAGING
|
||||
const mongoUriProd = process.env.MONGODB_URI_PROD
|
||||
let throttleMessageFetching = true
|
||||
const APP_ENV = process.env.POTION_APP_ENV
|
||||
|
||||
const cloudFrontUrlProd = process.env.CLOUDFRONT_URL_PROD
|
||||
const cloudFrontUrlDev = process.env.CLOUDFRONT_URL_DEV
|
||||
const cloudFrontUrlStaging = process.env.CLOUDFRONT_URL_STAGING
|
||||
|
||||
const updateVoiceCloning = async (data) => {
|
||||
const updated = await voiceCloningService.update(data)
|
||||
if (!updated) {
|
||||
throw new Error(`Voice cloning job ${data._id} was not found`)
|
||||
}
|
||||
return updated
|
||||
}
|
||||
|
||||
const updateUserAudioProfile = async (data) => {
|
||||
const updated = await userAudioProfileService.update(data)
|
||||
if (!updated) {
|
||||
throw new Error(`User audio profile ${data._id} was not found`)
|
||||
}
|
||||
return updated
|
||||
}
|
||||
|
||||
const updateUrl = (str, cloudFrontUrl) => {
|
||||
if (!cloudFrontUrl) return str
|
||||
const host = new URL(str).host
|
||||
return str.replace(`https://${host}`, cloudFrontUrl)
|
||||
}
|
||||
|
||||
function connectDB(dbUri, retryCount = 0) {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log('Connection Attempt : ', retryCount)
|
||||
mongoose.set('strictQuery', true)
|
||||
mongoose
|
||||
.connect(dbUri)
|
||||
.then((msg) => {
|
||||
console.log('Connected to Mongo DB !')
|
||||
resolve()
|
||||
})
|
||||
.catch((err) => {
|
||||
console.log('Failed to connect dns mongo: ', err)
|
||||
if (retryCount < 6) {
|
||||
resolve(connectDB(dbUri, retryCount + 1))
|
||||
} else {
|
||||
reject(err)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function execShellCommand(cmd, logPath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
exec(
|
||||
cmd,
|
||||
{ maxBuffer: 1024 * 1000000 },
|
||||
(error, stdout = '', stderr = '') => {
|
||||
Promise.all([
|
||||
fs.promises.writeFile(`${logPath}/error.log`, stderr),
|
||||
fs.promises.writeFile(`${logPath}/info.log`, stdout),
|
||||
])
|
||||
.then(() => {
|
||||
if (error) {
|
||||
console.log('Error while processing python command', error)
|
||||
reject(error)
|
||||
return
|
||||
}
|
||||
resolve({ stdout, stderr })
|
||||
})
|
||||
.catch(reject)
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async function getFile(waveUrl, path) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = https.get(waveUrl, (res) => {
|
||||
if (res.statusCode < 200 || res.statusCode >= 300) {
|
||||
res.resume()
|
||||
reject(
|
||||
new Error(`Unable to download training audio: HTTP ${res.statusCode}`)
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
const writeStream = fs.createWriteStream(path)
|
||||
|
||||
res.pipe(writeStream)
|
||||
res.on('error', reject)
|
||||
writeStream.on('error', reject)
|
||||
|
||||
writeStream.on('finish', () => {
|
||||
writeStream.close()
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
request.on('error', reject)
|
||||
})
|
||||
}
|
||||
|
||||
function pad(s) {
|
||||
while (s.length < 3) s = '0' + s // IN future we will need padding to 4
|
||||
return s
|
||||
}
|
||||
|
||||
const processQueue = () => {
|
||||
/* eslint-disable no-async-promise-executor */
|
||||
return new Promise(async (resolve, reject) => {
|
||||
try {
|
||||
const response = await sqs.fetchMessageFromSQS(sqsQueueUrl)
|
||||
|
||||
if (
|
||||
typeof response.Messages !== 'undefined' &&
|
||||
response.Messages.length > 0
|
||||
) {
|
||||
throttleMessageFetching = false
|
||||
const envelope = JSON.parse(response.Messages[0].Body)
|
||||
const job = parseJobEnvelope(envelope)
|
||||
const receiptHandle = response.Messages[0].ReceiptHandle
|
||||
console.log('job===', job)
|
||||
|
||||
const { metadata, input, _id, userAudioProfileId, tier } = job
|
||||
const tierConfig = resolveTierConfig(tier)
|
||||
console.log('userAudioProfileId', userAudioProfileId)
|
||||
console.log('_id', _id)
|
||||
const env = job.env || APP_ENV || 'development'
|
||||
console.log('env', env)
|
||||
console.log('tier', tierConfig.tier)
|
||||
|
||||
console.log('metadata------', metadata)
|
||||
console.log('input', input)
|
||||
const DB_URI =
|
||||
env === 'production'
|
||||
? mongoUriProd
|
||||
: env === 'staging'
|
||||
? mongoUriStaging
|
||||
: mongoUriDev
|
||||
|
||||
console.log('DB_URI ', DB_URI)
|
||||
await connectDB(DB_URI)
|
||||
|
||||
const cloudFrontUrl =
|
||||
env === 'production'
|
||||
? cloudFrontUrlProd
|
||||
: env === 'staging'
|
||||
? cloudFrontUrlStaging
|
||||
: cloudFrontUrlDev
|
||||
|
||||
try {
|
||||
const { directoryName } = metadata
|
||||
console.log('directoryName', directoryName)
|
||||
const logPath = `/mnt/efs/potion-voice/${env}/${directoryName}`
|
||||
if (!fs.existsSync(logPath)) {
|
||||
fs.mkdirSync(logPath, { recursive: true })
|
||||
}
|
||||
// update the db model to processing
|
||||
await updateVoiceCloning({
|
||||
_id,
|
||||
status: 'processing',
|
||||
tier: tierConfig.tier,
|
||||
})
|
||||
await updateUserAudioProfile({
|
||||
_id: userAudioProfileId,
|
||||
status: 'processing',
|
||||
})
|
||||
|
||||
// create directory for userid-useraudioprofileid if not exist
|
||||
const rootPath = `/tmp/${directoryName}`
|
||||
const wavePath = `${rootPath}/wav48/1`
|
||||
if (!fs.existsSync(wavePath)) {
|
||||
fs.mkdirSync(wavePath, { recursive: true })
|
||||
}
|
||||
|
||||
const txtPath = `${rootPath}/txt/1`
|
||||
if (!fs.existsSync(txtPath)) {
|
||||
fs.mkdirSync(txtPath, { recursive: true })
|
||||
}
|
||||
// download the training data files and put it in respective directories
|
||||
for (let index = 0; index < input.length; index++) {
|
||||
const item = input[index]
|
||||
|
||||
const { waveUrl, originalText } = item
|
||||
// download wave file
|
||||
const waveFilePath = `${wavePath}/1_${pad('' + (index + 1))}.wav`
|
||||
|
||||
await getFile(updateUrl(waveUrl, cloudFrontUrl), waveFilePath)
|
||||
|
||||
const txtFilePath = `${txtPath}/1_${pad('' + (index + 1))}.txt`
|
||||
await fs.promises.writeFile(txtFilePath, originalText)
|
||||
}
|
||||
|
||||
const zipFileName = directoryName + '.tgz'
|
||||
|
||||
// /tmp/directoryName.tgz
|
||||
|
||||
await execShellCommand(
|
||||
`cd /tmp && tar czvf ${zipFileName} ${directoryName}`,
|
||||
logPath
|
||||
)
|
||||
console.log('ZIP created ', zipFileName)
|
||||
|
||||
// re-sample audio
|
||||
const SAMPLING_LABEL = `Time Taken for re-sampling ${directoryName}`
|
||||
console.time(SAMPLING_LABEL)
|
||||
|
||||
const outputPath = `/mnt/efs/potion-voice/${env}/${directoryName}`
|
||||
|
||||
const samplingCommand = `python3 ../voice-cloning/prepare_datasets.py --dataset_preset ${tierConfig.datasetPreset} --dataset_archive_path /tmp/${zipFileName} --output_path ${outputPath}`
|
||||
console.log('samplingCommand ', samplingCommand)
|
||||
const samplingResponse = await execShellCommand(
|
||||
samplingCommand,
|
||||
logPath
|
||||
)
|
||||
console.timeEnd(SAMPLING_LABEL)
|
||||
|
||||
// /mnt/efs/potion-voice/${env}/speakrs.pth
|
||||
// /mnt/efs/potion-voice/${env}/txt
|
||||
// /mnt/efs/potion-voice/${env}/${directoryName}/wav
|
||||
|
||||
const outPath = `/mnt/efs/potion-voice/${env}/${directoryName}/sr22050/${directoryName}`
|
||||
|
||||
const resultsPath = outPath + '/results'
|
||||
|
||||
//update pth file for cloning
|
||||
// clone the voice
|
||||
const VOICE_CLONING_LABEL = `Time Taken for voice cloning ${directoryName}`
|
||||
console.time(VOICE_CLONING_LABEL)
|
||||
const trainingModelCommand = `python3 ../voice-cloning/clone_voice.py --baseline_model_path ${tierConfig.baselineModelPath} --speaker_dataset_path ${outPath} --speaker_embeddings_path ${
|
||||
outPath + '/speakers.pth'
|
||||
} --output_path ${resultsPath}`
|
||||
|
||||
console.log('Training Model Command', trainingModelCommand)
|
||||
const trainingResponse = await execShellCommand(
|
||||
trainingModelCommand,
|
||||
logPath
|
||||
)
|
||||
|
||||
console.timeEnd(VOICE_CLONING_LABEL)
|
||||
|
||||
let generatedDirectoryName = ''
|
||||
fs.readdirSync(`${resultsPath}/`).forEach((file) => {
|
||||
if (file.includes('vits_potion_clone'))
|
||||
// use output from above to get right path and directory name
|
||||
generatedDirectoryName = file
|
||||
})
|
||||
if (!generatedDirectoryName) {
|
||||
throw new Error(
|
||||
`Voice cloning did not produce a model directory for tier ${tierConfig.tier}`
|
||||
)
|
||||
}
|
||||
|
||||
// minimize cloning model
|
||||
const VOICE_MINIMIZE_LABEL = `Time Taken for voice minimizing cloning ${directoryName}`
|
||||
console.time(VOICE_MINIMIZE_LABEL)
|
||||
const minimizeCloningModelCommand = `python3 ../voice-cloning/minimize_cloned_voice_model.py --voice_model_asset_path ${
|
||||
resultsPath + '/' + generatedDirectoryName + '/'
|
||||
} --voice_model_name ${tierConfig.checkpointName}`
|
||||
|
||||
console.log(
|
||||
'Minimize Cloning Model Command',
|
||||
minimizeCloningModelCommand
|
||||
)
|
||||
const minimizeCloning = await execShellCommand(
|
||||
minimizeCloningModelCommand,
|
||||
logPath
|
||||
)
|
||||
console.timeEnd(VOICE_MINIMIZE_LABEL)
|
||||
|
||||
const lightCheckpointName = tierConfig.checkpointName.endsWith('.pth')
|
||||
? tierConfig.checkpointName.replace(/\.pth$/, '_light.pth')
|
||||
: `${tierConfig.checkpointName}_light`
|
||||
|
||||
const training_model_path = {
|
||||
voice_model_path: `${resultsPath}/${generatedDirectoryName}/${tierConfig.checkpointName}`,
|
||||
voice_model_config_path: `${resultsPath}/${generatedDirectoryName}/config.json`,
|
||||
voice_model_speakers_file_path: `${outPath}/speakers.pth`, // TODO update the name to voice model speakers embeddings
|
||||
voice_model_light_path: `${resultsPath}/${generatedDirectoryName}/${lightCheckpointName}`,
|
||||
voice_model_config_light_path: `${resultsPath}/${generatedDirectoryName}/config_light.json`,
|
||||
}
|
||||
|
||||
// add code to put that model into S3
|
||||
const keys = Object.keys(training_model_path)
|
||||
|
||||
const training_model_s3_path = {}
|
||||
|
||||
for (let index = 0; index < keys.length; index++) {
|
||||
const path = training_model_path[keys[index]]
|
||||
const s3Path = await s3.upload({
|
||||
filePath: path,
|
||||
fileName: `${directoryName}/${path.split('/').pop()}`,
|
||||
bucket: `potion-voice-users-training-model/${env}`,
|
||||
})
|
||||
training_model_s3_path[keys[index]] = s3Path
|
||||
}
|
||||
// add S3 path to user audio profile model
|
||||
await updateUserAudioProfile({
|
||||
_id: userAudioProfileId,
|
||||
status: 'completed',
|
||||
training_model_path,
|
||||
training_model_s3_path,
|
||||
})
|
||||
await updateVoiceCloning({
|
||||
_id,
|
||||
status: 'completed',
|
||||
tier: tierConfig.tier,
|
||||
training_model: training_model_s3_path,
|
||||
})
|
||||
|
||||
// Acknowledge only after the model and terminal state are durable.
|
||||
await sqs.deleteMessageFromSQS(sqsQueueUrl, receiptHandle)
|
||||
} catch (error) {
|
||||
console.log('error********************', error)
|
||||
Bugsnag.notify(
|
||||
new Error(
|
||||
`Unable to train for voice cloning videos ` + JSON.stringify(job)
|
||||
)
|
||||
)
|
||||
Bugsnag.notify(error)
|
||||
|
||||
// update the db to set status as error
|
||||
await updateVoiceCloning({
|
||||
_id,
|
||||
status: 'error',
|
||||
tier: tierConfig.tier,
|
||||
})
|
||||
await updateUserAudioProfile({
|
||||
_id: userAudioProfileId,
|
||||
status: 'error',
|
||||
})
|
||||
|
||||
resolve() // to continue working on new jobs
|
||||
}
|
||||
} else {
|
||||
throttleMessageFetching = true
|
||||
}
|
||||
resolve()
|
||||
} catch (error) {
|
||||
console.error('Error while training voice clone', { error })
|
||||
Bugsnag.notify(error)
|
||||
resolve() // to continue working on new jobs
|
||||
} finally {
|
||||
mongoose.connection.close()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms)
|
||||
})
|
||||
}
|
||||
const init = async () => {
|
||||
console.log('potion Voice Clone Process Started')
|
||||
Bugsnag.start({
|
||||
appVersion: APP_ENV + version,
|
||||
apiKey: process.env.BUGSNAG_BACKEND_KEY,
|
||||
releaseStage: process.env.NODE_ENV,
|
||||
})
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
await processQueue()
|
||||
if (throttleMessageFetching) await sleep(2000)
|
||||
}
|
||||
} catch (error) {
|
||||
Bugsnag.notify(error)
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) init()
|
||||
|
||||
module.exports = {
|
||||
init,
|
||||
processQueue,
|
||||
updateUserAudioProfile,
|
||||
updateVoiceCloning,
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"name": "voice-cloning-job-handler",
|
||||
"version": "1.0.0",
|
||||
"description": "This will handle the voice cloning jobs",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "node ../test/job_contract.test.js",
|
||||
"deploy-production": "npx dotenv-cli -e ./app-scripts/env-aws-code-deploy/.env.production.aws-code-deploy node ./app-scripts/deploy-scripts/deploy-production.js",
|
||||
"deploy-staging": "npx dotenv-cli -e ./app-scripts/env-aws-code-deploy/.env.staging.aws-code-deploy node ./app-scripts/deploy-scripts/deploy-staging.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@bugsnag/js": "^7.3.5",
|
||||
"aws-sdk": "^2.752.0",
|
||||
"fs-extra": "^9.0.1",
|
||||
"mongoose": "^6.8.0",
|
||||
"pm2": "^5.2.0",
|
||||
"rimraf": "^3.0.2",
|
||||
"uuid": "^8.3.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"aws-code-deploy": "^1.0.11"
|
||||
},
|
||||
"author": "potion Team",
|
||||
"license": "ISC"
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
'use strict'
|
||||
|
||||
const DEFAULT_TIER = 'legacy'
|
||||
const PRO_V2_TIER = 'pro_v2'
|
||||
|
||||
const isObject = (value) =>
|
||||
value !== null && typeof value === 'object' && !Array.isArray(value)
|
||||
|
||||
const firstPresent = (...values) =>
|
||||
values.find(
|
||||
(value) => value !== undefined && value !== null && value !== ''
|
||||
)
|
||||
|
||||
const normalizeTier = (tier) => {
|
||||
if (tier === undefined || tier === null || tier === '') return DEFAULT_TIER
|
||||
if (typeof tier !== 'string') {
|
||||
throw new TypeError('Voice cloning tier must be a string')
|
||||
}
|
||||
|
||||
return tier.trim().toLowerCase() || DEFAULT_TIER
|
||||
}
|
||||
|
||||
const unwrapJob = (envelope) => {
|
||||
if (!isObject(envelope)) {
|
||||
throw new TypeError('Voice cloning queue message must be an object')
|
||||
}
|
||||
|
||||
// Older producers spread a Mongoose document into the SQS envelope, which
|
||||
// puts the useful fields under `_doc`. Newer producers send a plain job (or
|
||||
// put that job under `job`/`payload`). Keep both contracts consumable.
|
||||
const candidates = [
|
||||
envelope._doc,
|
||||
isObject(envelope.job) && envelope.job._doc,
|
||||
envelope.job,
|
||||
isObject(envelope.payload) && envelope.payload._doc,
|
||||
envelope.payload,
|
||||
isObject(envelope.data) && envelope.data._doc,
|
||||
envelope.data,
|
||||
envelope,
|
||||
]
|
||||
|
||||
const payload = candidates.find(isObject)
|
||||
if (!payload) throw new TypeError('Voice cloning job payload is missing')
|
||||
|
||||
return payload
|
||||
}
|
||||
|
||||
const parseJobEnvelope = (envelope) => {
|
||||
const payload = unwrapJob(envelope)
|
||||
const metadata = firstPresent(payload.metadata, envelope.metadata, null)
|
||||
const metadataObject = isObject(metadata) ? metadata : {}
|
||||
|
||||
return {
|
||||
...payload,
|
||||
_id: firstPresent(payload._id, payload.id, envelope._id, envelope.id),
|
||||
userAudioProfileId: firstPresent(
|
||||
payload.userAudioProfileId,
|
||||
payload.user_audio_profile_id,
|
||||
envelope.userAudioProfileId,
|
||||
envelope.user_audio_profile_id
|
||||
),
|
||||
env: firstPresent(
|
||||
payload.env,
|
||||
payload.environment,
|
||||
envelope.env,
|
||||
envelope.environment
|
||||
),
|
||||
metadata,
|
||||
tier: normalizeTier(
|
||||
firstPresent(payload.tier, envelope.tier, metadataObject.tier)
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
const resolveTierConfig = (tier, environment = process.env) => {
|
||||
const normalizedTier = normalizeTier(tier)
|
||||
const isProV2 = normalizedTier === PRO_V2_TIER
|
||||
|
||||
return {
|
||||
tier: normalizedTier,
|
||||
datasetPreset:
|
||||
(isProV2 && environment.PRO_V2_DATASET_PRESET) ||
|
||||
environment.VOICE_CLONING_DATASET_PRESET ||
|
||||
'potion_voice_cloning',
|
||||
baselineModelPath:
|
||||
(isProV2 && environment.PRO_V2_BASELINE_MODEL_PATH) ||
|
||||
environment.VOICE_CLONING_BASELINE_MODEL_PATH ||
|
||||
'../voice-cloning/pretrained-models/checkpoint_365000.pth',
|
||||
checkpointName:
|
||||
(isProV2 && environment.PRO_V2_CHECKPOINT_NAME) ||
|
||||
environment.VOICE_CLONING_CHECKPOINT_NAME ||
|
||||
'checkpoint_365200.pth',
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
DEFAULT_TIER,
|
||||
PRO_V2_TIER,
|
||||
normalizeTier,
|
||||
parseJobEnvelope,
|
||||
resolveTierConfig,
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
const mongoose = require('mongoose')
|
||||
const Schema = mongoose.Schema
|
||||
|
||||
const VoiceCloningSchema = Schema(
|
||||
{
|
||||
userId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: 'User',
|
||||
required: true,
|
||||
},
|
||||
userAudioProfileId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: 'UserAudioProfile',
|
||||
required: true,
|
||||
},
|
||||
status: {
|
||||
type: String,
|
||||
required: false,
|
||||
default: 'created',
|
||||
},
|
||||
tier: {
|
||||
type: String,
|
||||
required: false,
|
||||
trim: true,
|
||||
lowercase: true,
|
||||
default: null,
|
||||
},
|
||||
input: {
|
||||
type: Schema.Types.Mixed,
|
||||
default: null,
|
||||
},
|
||||
training_model: {
|
||||
type: Schema.Types.Mixed,
|
||||
default: null,
|
||||
},
|
||||
metadata: {
|
||||
type: Schema.Types.Mixed,
|
||||
default: null,
|
||||
},
|
||||
deleted: {
|
||||
type: Boolean,
|
||||
required: true,
|
||||
default: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
timestamps: true,
|
||||
}
|
||||
)
|
||||
|
||||
module.exports =
|
||||
mongoose.models.VoiceCloning ||
|
||||
mongoose.model('VoiceCloning', VoiceCloningSchema)
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"captured_at": "2026-09-29T23:08:30Z",
|
||||
"grader_mode": "agentic",
|
||||
"grader_model": "claude-fable-5-1",
|
||||
"grader_samples": "1",
|
||||
"grading_standard": "consolidated",
|
||||
"grader_prompt_file": "grader-system-prompt-consolidated.md",
|
||||
"grader_prompt_sha256": "85edbe174ef7af339641db1f808cd4da3b2d3ee9587a07ac4e3a593011748dc9",
|
||||
"grader_guidance_file": "holistic-rubric.md",
|
||||
"grader_guidance_sha256": "316afb4138ddd686d3f73b3ab85c456e45117eb8df11b79bd39f3aa50fd3cf11",
|
||||
"render_grade_file": "render-grade-consolidated.py",
|
||||
"render_grade_sha256": "db8b668c536007abbd7d9719dc08dd388507e67df7da08f63bc8c495d58840cb"
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
Captured 7 agent output files
|
||||
Launching Claude Code grader (requested model: claude-fable-5-1, samples: 1)...
|
||||
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"id": "18327e53-0e9a-4a10-92ec-43208f719152",
|
||||
"started_at": "2026-09-29T23:08:21.600933",
|
||||
"updated_at": "2026-09-29T23:10:41.543371Z",
|
||||
"finished_at": null,
|
||||
"n_total_trials": 1,
|
||||
"stats": {
|
||||
"n_completed_trials": 1,
|
||||
"n_errored_trials": 1,
|
||||
"n_running_trials": 0,
|
||||
"n_pending_trials": 0,
|
||||
"n_cancelled_trials": 1,
|
||||
"n_retries": 0,
|
||||
"evals": {
|
||||
"replay__adhoc": {
|
||||
"n_trials": 0,
|
||||
"n_errors": 1,
|
||||
"metrics": [
|
||||
{
|
||||
"mean": 0.0
|
||||
}
|
||||
],
|
||||
"pass_at_k": {},
|
||||
"reward_stats": {},
|
||||
"exception_stats": {
|
||||
"CancelledError": [
|
||||
"mishandled_pro_v2__8qttC5x"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"n_input_tokens": null,
|
||||
"n_cache_tokens": null,
|
||||
"n_output_tokens": null,
|
||||
"cost_usd": null
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user