Hugging Face's logo

Spaces:
ArchCoder
/
federated-credit-scoring
Running

App Files Community
“Transcendental-Programmer” commited on
Commit
fc5fa78
·
1 Parent(s): 76abe40

fix : minor fixes

Browse files
Files changed (18) hide show
  1. config/client_config.yaml +12 -2
  2. config/client_config_2.yaml +26 -0
  3. config/server_config.yaml +20 -1
  4. logs/federated_learning.log +552 -0
  5. notebooks/data_exploration.ipynb +6 -0
  6. notebooks/model_evaluation.ipynb +6 -0
  7. requirements.txt +5 -1
  8. run_two_clients.sh +11 -0
  9. src/api/__init__.py +0 -0
  10. src/api/client.py +156 -0
  11. src/api/server.py +165 -0
  12. src/client/model.py +141 -51
  13. src/client/model_new.py +214 -0
  14. src/main.py +7 -3
  15. src/server/aggregator.py +28 -12
  16. src/server/coordinator.py +140 -38
  17. src/utils/metrics.py +7 -0
  18. test_implementation.py +157 -0
config/client_config.yaml CHANGED
@@ -1,11 +1,17 @@
1
  # client_config.yaml configuration
2
 
3
  client:
 
 
 
 
4
  # Data configuration
5
  data:
6
  batch_size: 32
7
  shuffle_buffer: 1000
 
8
  input_dim: 32
 
9
 
10
  # Model configuration
11
  model:
@@ -15,9 +21,13 @@ client:
15
 
16
  # Training configuration
17
  training:
18
- local_epochs: 5
19
  learning_rate: 0.001
20
 
 
 
 
 
 
21
  monitoring:
22
  log_level: "INFO"
23
-
 
1
  # client_config.yaml configuration
2
 
3
  client:
4
+ # Client identification
5
+ id: "client_1"
6
+ server_url: "http://localhost:8080"
7
+
8
  # Data configuration
9
  data:
10
  batch_size: 32
11
  shuffle_buffer: 1000
12
+ prefetch_buffer: 10
13
  input_dim: 32
14
+ dataset_size: 100
15
 
16
  # Model configuration
17
  model:
 
21
 
22
  # Training configuration
23
  training:
24
+ local_epochs: 3
25
  learning_rate: 0.001
26
 
27
+ # Privacy configuration
28
+ privacy:
29
+ differential_privacy: false
30
+ noise_multiplier: 0.1
31
+
32
  monitoring:
33
  log_level: "INFO"
 
config/client_config_2.yaml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ client:
2
+ id: "client_2"
3
+ server_url: "http://localhost:8080"
4
+
5
+ data:
6
+ batch_size: 32
7
+ shuffle_buffer: 1000
8
+ prefetch_buffer: 10
9
+ input_dim: 32
10
+ dataset_size: 100
11
+
12
+ model:
13
+ type: "feedforward"
14
+ hidden_dims: [128, 64]
15
+ activation: "relu"
16
+
17
+ training:
18
+ local_epochs: 3
19
+ learning_rate: 0.001
20
+
21
+ privacy:
22
+ differential_privacy: false
23
+ noise_multiplier: 0.1
24
+
25
+ monitoring:
26
+ log_level: "INFO"
config/server_config.yaml CHANGED
@@ -1,9 +1,15 @@
1
  # server_config.yaml configuration
2
 
3
  server:
 
 
 
 
 
 
4
  # Federated learning configuration
5
  federated:
6
- min_clients: 1
7
  rounds: 10
8
  sample_fraction: 0.8
9
 
@@ -16,3 +22,16 @@ server:
16
  monitoring:
17
  log_level: "INFO"
18
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  # server_config.yaml configuration
2
 
3
  server:
4
+ # API server configuration
5
+ api:
6
+ host: "0.0.0.0"
7
+ port: 8080
8
+ debug: false
9
+
10
  # Federated learning configuration
11
  federated:
12
+ min_clients: 2
13
  rounds: 10
14
  sample_fraction: 0.8
15
 
 
22
  monitoring:
23
  log_level: "INFO"
24
 
25
+ # Model configuration
26
+ model:
27
+ architecture: "simple_nn"
28
+ input_dim: 32
29
+ hidden_layers: [128, 64]
30
+ output_dim: 1
31
+
32
+ # Training configuration
33
+ training:
34
+ learning_rate: 0.001
35
+ batch_size: 32
36
+ local_epochs: 3
37
+
logs/federated_learning.log CHANGED
@@ -336,3 +336,555 @@ Round 1/10
336
  Round 1/10
337
  2024-12-17 21:14:04,835 - src.server.coordinator - INFO - ------------------------------
338
  2024-12-17 21:14:04,836 - src.server.coordinator - WARNING - Waiting for clients... (active: 0/1)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
336
  Round 1/10
337
  2024-12-17 21:14:04,835 - src.server.coordinator - INFO - ------------------------------
338
  2024-12-17 21:14:04,836 - src.server.coordinator - WARNING - Waiting for clients... (active: 0/1)
339
+ 2025-06-27 15:49:52,624 - __main__ - INFO -
340
+ ==================================================
341
+ 2025-06-27 15:49:52,626 - __main__ - INFO - New Training Session Started
342
+ 2025-06-27 15:49:52,627 - __main__ - INFO - ==================================================
343
+
344
+ 2025-06-27 15:49:52,627 - src.server.aggregator - INFO - FederatedAggregator initialized. Weighted: True
345
+ 2025-06-27 15:49:52,627 - src.server.coordinator - INFO - FederatedCoordinator initialized.
346
+ 2025-06-27 15:49:52,627 - __main__ - INFO - Starting federated server...
347
+ 2025-06-27 15:49:52,650 - src.server.coordinator - INFO -
348
+ ============================================================
349
+ 2025-06-27 15:49:52,650 - src.server.coordinator - INFO - Federated Learning Server Starting
350
+ 2025-06-27 15:49:52,651 - src.server.coordinator - INFO - ============================================================
351
+ 2025-06-27 15:49:52,651 - src.server.coordinator - INFO -
352
+ Server Configuration:
353
+ 2025-06-27 15:49:52,651 - src.server.coordinator - INFO - ------------------------------
354
+ 2025-06-27 15:49:52,652 - src.server.coordinator - INFO - Minimum clients required: 2
355
+ 2025-06-27 15:49:52,653 - src.server.coordinator - INFO - Total rounds planned: 10
356
+ 2025-06-27 15:49:52,653 - src.server.coordinator - INFO - Current active clients: 0
357
+ 2025-06-27 15:49:52,653 - src.server.coordinator - INFO - ------------------------------
358
+
359
+ 2025-06-27 15:49:53,212 - src.api.server - INFO - Federated API server started in background on 0.0.0.0:8080
360
+ 2025-06-27 15:49:53,212 - src.server.coordinator - INFO - API server started on 0.0.0.0:8080
361
+ 2025-06-27 15:49:53,439 - werkzeug - INFO - WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
362
+ * Running on all addresses (0.0.0.0)
363
+ * Running on http://127.0.0.1:8080
364
+ * Running on http://192.168.68.72:8080
365
+ 2025-06-27 15:49:53,440 - werkzeug - INFO - Press CTRL+C to quit
366
+ 2025-06-27 15:50:11,506 - __main__ - INFO -
367
+ ==================================================
368
+ 2025-06-27 15:50:11,507 - __main__ - INFO - New Training Session Started
369
+ 2025-06-27 15:50:11,507 - __main__ - INFO - ==================================================
370
+
371
+ 2025-06-27 15:50:11,692 - __main__ - INFO - Starting federated client with ID: client_1
372
+ 2025-06-27 15:50:11,693 - src.client.model - INFO - Client client_1 starting...
373
+ 2025-06-27 15:50:11,758 - src.api.client - INFO - Waiting for server at http://localhost:8080...
374
+ 2025-06-27 15:50:16,772 - src.api.client - INFO - Waiting for server at http://localhost:8080...
375
+ 2025-06-27 15:50:21,780 - src.api.client - INFO - Waiting for server at http://localhost:8080...
376
+ 2025-06-27 15:50:26,788 - src.api.client - INFO - Waiting for server at http://localhost:8080...
377
+ 2025-06-27 15:50:33,840 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:50:33] "GET /health HTTP/1.1" 200 -
378
+ 2025-06-27 15:50:33,842 - src.api.client - INFO - Server is available at http://localhost:8080
379
+ 2025-06-27 15:50:35,917 - src.server.coordinator - INFO - Client client_1 registered successfully
380
+ 2025-06-27 15:50:35,918 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:50:35] "POST /register HTTP/1.1" 200 -
381
+ 2025-06-27 15:50:35,919 - src.api.client - INFO - Client client_1 registered successfully
382
+ 2025-06-27 15:50:35,920 - src.client.model - INFO - Successfully registered with server
383
+ 2025-06-27 15:50:35,920 - src.client.model - INFO - Dataset size: 100
384
+ 2025-06-27 15:50:35,921 - src.client.model - INFO - Model parameters: 12,545
385
+ 2025-06-27 15:50:37,951 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:50:37] "GET /training_status HTTP/1.1" 200 -
386
+ 2025-06-27 15:50:45,000 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:50:45] "GET /training_status HTTP/1.1" 200 -
387
+ 2025-06-27 15:50:52,045 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:50:52] "GET /training_status HTTP/1.1" 200 -
388
+ 2025-06-27 15:50:59,091 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:50:59] "GET /training_status HTTP/1.1" 200 -
389
+ 2025-06-27 15:51:06,134 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:51:06] "GET /training_status HTTP/1.1" 200 -
390
+ 2025-06-27 15:51:13,174 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:51:13] "GET /training_status HTTP/1.1" 200 -
391
+ 2025-06-27 15:51:20,204 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:51:20] "GET /training_status HTTP/1.1" 200 -
392
+ 2025-06-27 15:51:27,245 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:51:27] "GET /training_status HTTP/1.1" 200 -
393
+ 2025-06-27 15:51:34,306 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:51:34] "GET /training_status HTTP/1.1" 200 -
394
+ 2025-06-27 15:51:41,340 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:51:41] "GET /training_status HTTP/1.1" 200 -
395
+ 2025-06-27 15:51:48,388 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:51:48] "GET /training_status HTTP/1.1" 200 -
396
+ 2025-06-27 15:51:55,441 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:51:55] "GET /training_status HTTP/1.1" 200 -
397
+ 2025-06-27 15:52:02,484 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:52:02] "GET /training_status HTTP/1.1" 200 -
398
+ 2025-06-27 15:52:09,513 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:52:09] "GET /training_status HTTP/1.1" 200 -
399
+ 2025-06-27 15:52:16,574 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:52:16] "GET /training_status HTTP/1.1" 200 -
400
+ 2025-06-27 15:52:23,606 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:52:23] "GET /training_status HTTP/1.1" 200 -
401
+ 2025-06-27 15:52:30,641 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:52:30] "GET /training_status HTTP/1.1" 200 -
402
+ 2025-06-27 15:52:37,679 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:52:37] "GET /training_status HTTP/1.1" 200 -
403
+ 2025-06-27 15:52:44,770 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:52:44] "GET /training_status HTTP/1.1" 200 -
404
+ 2025-06-27 15:52:51,830 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:52:51] "GET /training_status HTTP/1.1" 200 -
405
+ 2025-06-27 15:52:58,886 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:52:58] "GET /training_status HTTP/1.1" 200 -
406
+ 2025-06-27 15:53:05,936 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:53:05] "GET /training_status HTTP/1.1" 200 -
407
+ 2025-06-27 15:53:12,963 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:53:12] "GET /training_status HTTP/1.1" 200 -
408
+ 2025-06-27 15:53:20,012 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:53:20] "GET /training_status HTTP/1.1" 200 -
409
+ 2025-06-27 15:53:27,057 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:53:27] "GET /training_status HTTP/1.1" 200 -
410
+ 2025-06-27 15:53:34,097 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:53:34] "GET /training_status HTTP/1.1" 200 -
411
+ 2025-06-27 15:53:41,140 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:53:41] "GET /training_status HTTP/1.1" 200 -
412
+ 2025-06-27 15:53:48,205 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:53:48] "GET /training_status HTTP/1.1" 200 -
413
+ 2025-06-27 15:53:55,235 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:53:55] "GET /training_status HTTP/1.1" 200 -
414
+ 2025-06-27 15:54:02,275 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:54:02] "GET /training_status HTTP/1.1" 200 -
415
+ 2025-06-27 15:54:09,327 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:54:09] "GET /training_status HTTP/1.1" 200 -
416
+ 2025-06-27 15:54:16,367 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:54:16] "GET /training_status HTTP/1.1" 200 -
417
+ 2025-06-27 15:54:23,411 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:54:23] "GET /training_status HTTP/1.1" 200 -
418
+ 2025-06-27 15:54:30,445 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:54:30] "GET /training_status HTTP/1.1" 200 -
419
+ 2025-06-27 15:54:37,480 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:54:37] "GET /training_status HTTP/1.1" 200 -
420
+ 2025-06-27 15:54:44,531 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:54:44] "GET /training_status HTTP/1.1" 200 -
421
+ 2025-06-27 15:54:51,590 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:54:51] "GET /training_status HTTP/1.1" 200 -
422
+ 2025-06-27 15:54:58,628 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:54:58] "GET /training_status HTTP/1.1" 200 -
423
+ 2025-06-27 15:55:05,667 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:55:05] "GET /training_status HTTP/1.1" 200 -
424
+ 2025-06-27 15:55:12,701 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:55:12] "GET /training_status HTTP/1.1" 200 -
425
+ 2025-06-27 15:55:19,739 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:55:19] "GET /training_status HTTP/1.1" 200 -
426
+ 2025-06-27 15:55:26,782 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:55:26] "GET /training_status HTTP/1.1" 200 -
427
+ 2025-06-27 15:55:33,838 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:55:33] "GET /training_status HTTP/1.1" 200 -
428
+ 2025-06-27 15:55:40,881 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:55:40] "GET /training_status HTTP/1.1" 200 -
429
+ 2025-06-27 15:55:47,919 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:55:47] "GET /training_status HTTP/1.1" 200 -
430
+ 2025-06-27 15:55:49,344 - __main__ - INFO -
431
+ ==================================================
432
+ 2025-06-27 15:55:49,345 - __main__ - INFO - New Training Session Started
433
+ 2025-06-27 15:55:49,345 - __main__ - INFO - ==================================================
434
+
435
+ 2025-06-27 15:55:49,531 - __main__ - INFO - Starting federated client with ID: client_2
436
+ 2025-06-27 15:55:49,531 - src.client.model - INFO - Client client_2 starting...
437
+ 2025-06-27 15:55:51,578 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:55:51] "GET /health HTTP/1.1" 200 -
438
+ 2025-06-27 15:55:51,581 - src.api.client - INFO - Server is available at http://localhost:8080
439
+ 2025-06-27 15:55:53,649 - src.server.coordinator - INFO - Client client_2 registered successfully
440
+ 2025-06-27 15:55:53,649 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:55:53] "POST /register HTTP/1.1" 200 -
441
+ 2025-06-27 15:55:53,651 - src.api.client - INFO - Client client_2 registered successfully
442
+ 2025-06-27 15:55:53,652 - src.client.model - INFO - Successfully registered with server
443
+ 2025-06-27 15:55:53,652 - src.client.model - INFO - Dataset size: 100
444
+ 2025-06-27 15:55:53,652 - src.client.model - INFO - Model parameters: 12,545
445
+ 2025-06-27 15:55:54,962 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:55:54] "GET /training_status HTTP/1.1" 200 -
446
+ 2025-06-27 15:55:55,685 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:55:55] "GET /training_status HTTP/1.1" 200 -
447
+ 2025-06-27 15:56:02,006 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:02] "GET /training_status HTTP/1.1" 200 -
448
+ 2025-06-27 15:56:02,737 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:02] "GET /training_status HTTP/1.1" 200 -
449
+ 2025-06-27 15:56:09,045 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:09] "GET /training_status HTTP/1.1" 200 -
450
+ 2025-06-27 15:56:09,772 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:09] "GET /training_status HTTP/1.1" 200 -
451
+ 2025-06-27 15:56:16,093 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:16] "GET /training_status HTTP/1.1" 200 -
452
+ 2025-06-27 15:56:16,818 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:16] "GET /training_status HTTP/1.1" 200 -
453
+ 2025-06-27 15:56:23,145 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:23] "GET /training_status HTTP/1.1" 200 -
454
+ 2025-06-27 15:56:23,871 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:23] "GET /training_status HTTP/1.1" 200 -
455
+ 2025-06-27 15:56:30,192 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:30] "GET /training_status HTTP/1.1" 200 -
456
+ 2025-06-27 15:56:30,916 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:30] "GET /training_status HTTP/1.1" 200 -
457
+ 2025-06-27 15:56:37,227 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:37] "GET /training_status HTTP/1.1" 200 -
458
+ 2025-06-27 15:56:37,950 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:37] "GET /training_status HTTP/1.1" 200 -
459
+ 2025-06-27 15:56:44,273 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:44] "GET /training_status HTTP/1.1" 200 -
460
+ 2025-06-27 15:56:44,996 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:44] "GET /training_status HTTP/1.1" 200 -
461
+ 2025-06-27 15:56:51,318 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:51] "GET /training_status HTTP/1.1" 200 -
462
+ 2025-06-27 15:56:52,029 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:52] "GET /training_status HTTP/1.1" 200 -
463
+ 2025-06-27 15:56:58,370 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:58] "GET /training_status HTTP/1.1" 200 -
464
+ 2025-06-27 15:56:59,065 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:56:59] "GET /training_status HTTP/1.1" 200 -
465
+ 2025-06-27 15:57:05,413 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:05] "GET /training_status HTTP/1.1" 200 -
466
+ 2025-06-27 15:57:06,102 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:06] "GET /training_status HTTP/1.1" 200 -
467
+ 2025-06-27 15:57:12,446 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:12] "GET /training_status HTTP/1.1" 200 -
468
+ 2025-06-27 15:57:13,143 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:13] "GET /training_status HTTP/1.1" 200 -
469
+ 2025-06-27 15:57:19,482 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:19] "GET /training_status HTTP/1.1" 200 -
470
+ 2025-06-27 15:57:20,183 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:20] "GET /training_status HTTP/1.1" 200 -
471
+ 2025-06-27 15:57:26,523 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:26] "GET /training_status HTTP/1.1" 200 -
472
+ 2025-06-27 15:57:27,236 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:27] "GET /training_status HTTP/1.1" 200 -
473
+ 2025-06-27 15:57:33,568 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:33] "GET /training_status HTTP/1.1" 200 -
474
+ 2025-06-27 15:57:34,295 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:34] "GET /training_status HTTP/1.1" 200 -
475
+ 2025-06-27 15:57:40,613 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:40] "GET /training_status HTTP/1.1" 200 -
476
+ 2025-06-27 15:57:41,333 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:41] "GET /training_status HTTP/1.1" 200 -
477
+ 2025-06-27 15:57:47,660 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:47] "GET /training_status HTTP/1.1" 200 -
478
+ 2025-06-27 15:57:48,420 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:48] "GET /training_status HTTP/1.1" 200 -
479
+ 2025-06-27 15:57:54,722 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:54] "GET /training_status HTTP/1.1" 200 -
480
+ 2025-06-27 15:57:55,465 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:57:55] "GET /training_status HTTP/1.1" 200 -
481
+ 2025-06-27 15:58:01,758 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:01] "GET /training_status HTTP/1.1" 200 -
482
+ 2025-06-27 15:58:02,500 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:02] "GET /training_status HTTP/1.1" 200 -
483
+ 2025-06-27 15:58:08,774 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:08] "GET /training_status HTTP/1.1" 200 -
484
+ 2025-06-27 15:58:09,557 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:09] "GET /training_status HTTP/1.1" 200 -
485
+ 2025-06-27 15:58:15,807 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:15] "GET /training_status HTTP/1.1" 200 -
486
+ 2025-06-27 15:58:16,596 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:16] "GET /training_status HTTP/1.1" 200 -
487
+ 2025-06-27 15:58:22,838 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:22] "GET /training_status HTTP/1.1" 200 -
488
+ 2025-06-27 15:58:23,621 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:23] "GET /training_status HTTP/1.1" 200 -
489
+ 2025-06-27 15:58:29,871 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:29] "GET /training_status HTTP/1.1" 200 -
490
+ 2025-06-27 15:58:30,711 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:30] "GET /training_status HTTP/1.1" 200 -
491
+ 2025-06-27 15:58:36,926 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:36] "GET /training_status HTTP/1.1" 200 -
492
+ 2025-06-27 15:58:37,763 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:37] "GET /training_status HTTP/1.1" 200 -
493
+ 2025-06-27 15:58:43,963 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:43] "GET /training_status HTTP/1.1" 200 -
494
+ 2025-06-27 15:58:44,797 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:44] "GET /training_status HTTP/1.1" 200 -
495
+ 2025-06-27 15:58:51,009 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:51] "GET /training_status HTTP/1.1" 200 -
496
+ 2025-06-27 15:58:51,834 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:51] "GET /training_status HTTP/1.1" 200 -
497
+ 2025-06-27 15:58:58,063 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:58] "GET /training_status HTTP/1.1" 200 -
498
+ 2025-06-27 15:58:58,871 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:58:58] "GET /training_status HTTP/1.1" 200 -
499
+ 2025-06-27 15:59:05,100 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:05] "GET /training_status HTTP/1.1" 200 -
500
+ 2025-06-27 15:59:05,915 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:05] "GET /training_status HTTP/1.1" 200 -
501
+ 2025-06-27 15:59:12,148 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:12] "GET /training_status HTTP/1.1" 200 -
502
+ 2025-06-27 15:59:12,947 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:12] "GET /training_status HTTP/1.1" 200 -
503
+ 2025-06-27 15:59:19,184 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:19] "GET /training_status HTTP/1.1" 200 -
504
+ 2025-06-27 15:59:20,035 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:20] "GET /training_status HTTP/1.1" 200 -
505
+ 2025-06-27 15:59:26,238 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:26] "GET /training_status HTTP/1.1" 200 -
506
+ 2025-06-27 15:59:27,098 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:27] "GET /training_status HTTP/1.1" 200 -
507
+ 2025-06-27 15:59:33,279 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:33] "GET /training_status HTTP/1.1" 200 -
508
+ 2025-06-27 15:59:34,142 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:34] "GET /training_status HTTP/1.1" 200 -
509
+ 2025-06-27 15:59:40,328 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:40] "GET /training_status HTTP/1.1" 200 -
510
+ 2025-06-27 15:59:41,199 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:41] "GET /training_status HTTP/1.1" 200 -
511
+ 2025-06-27 15:59:47,377 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:47] "GET /training_status HTTP/1.1" 200 -
512
+ 2025-06-27 15:59:48,263 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:48] "GET /training_status HTTP/1.1" 200 -
513
+ 2025-06-27 15:59:54,428 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:54] "GET /training_status HTTP/1.1" 200 -
514
+ 2025-06-27 15:59:55,294 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 15:59:55] "GET /training_status HTTP/1.1" 200 -
515
+ 2025-06-27 16:00:01,493 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:01] "GET /training_status HTTP/1.1" 200 -
516
+ 2025-06-27 16:00:02,558 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:02] "GET /training_status HTTP/1.1" 200 -
517
+ 2025-06-27 16:00:08,531 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:08] "GET /training_status HTTP/1.1" 200 -
518
+ 2025-06-27 16:00:09,607 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:09] "GET /training_status HTTP/1.1" 200 -
519
+ 2025-06-27 16:00:15,585 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:15] "GET /training_status HTTP/1.1" 200 -
520
+ 2025-06-27 16:00:16,653 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:16] "GET /training_status HTTP/1.1" 200 -
521
+ 2025-06-27 16:00:22,627 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:22] "GET /training_status HTTP/1.1" 200 -
522
+ 2025-06-27 16:00:23,709 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:23] "GET /training_status HTTP/1.1" 200 -
523
+ 2025-06-27 16:00:29,674 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:29] "GET /training_status HTTP/1.1" 200 -
524
+ 2025-06-27 16:00:30,763 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:30] "GET /training_status HTTP/1.1" 200 -
525
+ 2025-06-27 16:00:36,721 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:36] "GET /training_status HTTP/1.1" 200 -
526
+ 2025-06-27 16:00:37,814 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:37] "GET /training_status HTTP/1.1" 200 -
527
+ 2025-06-27 16:00:43,753 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:43] "GET /training_status HTTP/1.1" 200 -
528
+ 2025-06-27 16:00:44,860 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:44] "GET /training_status HTTP/1.1" 200 -
529
+ 2025-06-27 16:00:50,799 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:50] "GET /training_status HTTP/1.1" 200 -
530
+ 2025-06-27 16:00:51,916 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:51] "GET /training_status HTTP/1.1" 200 -
531
+ 2025-06-27 16:00:57,860 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:57] "GET /training_status HTTP/1.1" 200 -
532
+ 2025-06-27 16:00:58,958 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:00:58] "GET /training_status HTTP/1.1" 200 -
533
+ 2025-06-27 16:01:04,907 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:04] "GET /training_status HTTP/1.1" 200 -
534
+ 2025-06-27 16:01:05,996 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:05] "GET /training_status HTTP/1.1" 200 -
535
+ 2025-06-27 16:01:11,956 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:11] "GET /training_status HTTP/1.1" 200 -
536
+ 2025-06-27 16:01:13,022 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:13] "GET /training_status HTTP/1.1" 200 -
537
+ 2025-06-27 16:01:18,989 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:18] "GET /training_status HTTP/1.1" 200 -
538
+ 2025-06-27 16:01:20,079 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:20] "GET /training_status HTTP/1.1" 200 -
539
+ 2025-06-27 16:01:26,028 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:26] "GET /training_status HTTP/1.1" 200 -
540
+ 2025-06-27 16:01:27,120 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:27] "GET /training_status HTTP/1.1" 200 -
541
+ 2025-06-27 16:01:33,073 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:33] "GET /training_status HTTP/1.1" 200 -
542
+ 2025-06-27 16:01:34,163 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:34] "GET /training_status HTTP/1.1" 200 -
543
+ 2025-06-27 16:01:40,124 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:40] "GET /training_status HTTP/1.1" 200 -
544
+ 2025-06-27 16:01:41,184 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:41] "GET /training_status HTTP/1.1" 200 -
545
+ 2025-06-27 16:01:47,179 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:47] "GET /training_status HTTP/1.1" 200 -
546
+ 2025-06-27 16:01:48,238 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:48] "GET /training_status HTTP/1.1" 200 -
547
+ 2025-06-27 16:01:54,226 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:54] "GET /training_status HTTP/1.1" 200 -
548
+ 2025-06-27 16:01:55,273 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:01:55] "GET /training_status HTTP/1.1" 200 -
549
+ 2025-06-27 16:02:01,284 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:01] "GET /training_status HTTP/1.1" 200 -
550
+ 2025-06-27 16:02:02,301 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:02] "GET /training_status HTTP/1.1" 200 -
551
+ 2025-06-27 16:02:08,338 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:08] "GET /training_status HTTP/1.1" 200 -
552
+ 2025-06-27 16:02:09,361 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:09] "GET /training_status HTTP/1.1" 200 -
553
+ 2025-06-27 16:02:15,398 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:15] "GET /training_status HTTP/1.1" 200 -
554
+ 2025-06-27 16:02:16,409 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:16] "GET /training_status HTTP/1.1" 200 -
555
+ 2025-06-27 16:02:22,464 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:22] "GET /training_status HTTP/1.1" 200 -
556
+ 2025-06-27 16:02:23,458 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:23] "GET /training_status HTTP/1.1" 200 -
557
+ 2025-06-27 16:02:29,494 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:29] "GET /training_status HTTP/1.1" 200 -
558
+ 2025-06-27 16:02:30,494 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:30] "GET /training_status HTTP/1.1" 200 -
559
+ 2025-06-27 16:02:36,566 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:36] "GET /training_status HTTP/1.1" 200 -
560
+ 2025-06-27 16:02:37,519 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:37] "GET /training_status HTTP/1.1" 200 -
561
+ 2025-06-27 16:02:43,609 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:43] "GET /training_status HTTP/1.1" 200 -
562
+ 2025-06-27 16:02:44,547 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:44] "GET /training_status HTTP/1.1" 200 -
563
+ 2025-06-27 16:02:50,664 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:50] "GET /training_status HTTP/1.1" 200 -
564
+ 2025-06-27 16:02:51,599 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:51] "GET /training_status HTTP/1.1" 200 -
565
+ 2025-06-27 16:02:57,721 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:57] "GET /training_status HTTP/1.1" 200 -
566
+ 2025-06-27 16:02:58,655 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:02:58] "GET /training_status HTTP/1.1" 200 -
567
+ 2025-06-27 16:03:04,774 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:04] "GET /training_status HTTP/1.1" 200 -
568
+ 2025-06-27 16:03:05,710 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:05] "GET /training_status HTTP/1.1" 200 -
569
+ 2025-06-27 16:03:11,822 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:11] "GET /training_status HTTP/1.1" 200 -
570
+ 2025-06-27 16:03:12,747 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:12] "GET /training_status HTTP/1.1" 200 -
571
+ 2025-06-27 16:03:18,859 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:18] "GET /training_status HTTP/1.1" 200 -
572
+ 2025-06-27 16:03:19,796 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:19] "GET /training_status HTTP/1.1" 200 -
573
+ 2025-06-27 16:03:25,911 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:25] "GET /training_status HTTP/1.1" 200 -
574
+ 2025-06-27 16:03:26,840 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:26] "GET /training_status HTTP/1.1" 200 -
575
+ 2025-06-27 16:03:32,933 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:32] "GET /training_status HTTP/1.1" 200 -
576
+ 2025-06-27 16:03:33,898 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:33] "GET /training_status HTTP/1.1" 200 -
577
+ 2025-06-27 16:03:40,003 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:40] "GET /training_status HTTP/1.1" 200 -
578
+ 2025-06-27 16:03:40,963 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:40] "GET /training_status HTTP/1.1" 200 -
579
+ 2025-06-27 16:03:47,047 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:47] "GET /training_status HTTP/1.1" 200 -
580
+ 2025-06-27 16:03:47,994 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:47] "GET /training_status HTTP/1.1" 200 -
581
+ 2025-06-27 16:03:54,094 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:54] "GET /training_status HTTP/1.1" 200 -
582
+ 2025-06-27 16:03:55,030 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:03:55] "GET /training_status HTTP/1.1" 200 -
583
+ 2025-06-27 16:04:01,155 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:01] "GET /training_status HTTP/1.1" 200 -
584
+ 2025-06-27 16:04:02,078 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:02] "GET /training_status HTTP/1.1" 200 -
585
+ 2025-06-27 16:04:08,196 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:08] "GET /training_status HTTP/1.1" 200 -
586
+ 2025-06-27 16:04:09,127 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:09] "GET /training_status HTTP/1.1" 200 -
587
+ 2025-06-27 16:04:15,242 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:15] "GET /training_status HTTP/1.1" 200 -
588
+ 2025-06-27 16:04:16,157 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:16] "GET /training_status HTTP/1.1" 200 -
589
+ 2025-06-27 16:04:22,291 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:22] "GET /training_status HTTP/1.1" 200 -
590
+ 2025-06-27 16:04:23,206 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:23] "GET /training_status HTTP/1.1" 200 -
591
+ 2025-06-27 16:04:29,329 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:29] "GET /training_status HTTP/1.1" 200 -
592
+ 2025-06-27 16:04:30,263 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:30] "GET /training_status HTTP/1.1" 200 -
593
+ 2025-06-27 16:04:36,363 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:36] "GET /training_status HTTP/1.1" 200 -
594
+ 2025-06-27 16:04:37,312 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:37] "GET /training_status HTTP/1.1" 200 -
595
+ 2025-06-27 16:04:43,401 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:43] "GET /training_status HTTP/1.1" 200 -
596
+ 2025-06-27 16:04:44,351 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:44] "GET /training_status HTTP/1.1" 200 -
597
+ 2025-06-27 16:04:50,453 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:50] "GET /training_status HTTP/1.1" 200 -
598
+ 2025-06-27 16:04:51,401 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:51] "GET /training_status HTTP/1.1" 200 -
599
+ 2025-06-27 16:04:57,510 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:57] "GET /training_status HTTP/1.1" 200 -
600
+ 2025-06-27 16:04:58,447 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:04:58] "GET /training_status HTTP/1.1" 200 -
601
+ 2025-06-27 16:05:04,548 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:04] "GET /training_status HTTP/1.1" 200 -
602
+ 2025-06-27 16:05:05,498 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:05] "GET /training_status HTTP/1.1" 200 -
603
+ 2025-06-27 16:05:11,583 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:11] "GET /training_status HTTP/1.1" 200 -
604
+ 2025-06-27 16:05:12,542 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:12] "GET /training_status HTTP/1.1" 200 -
605
+ 2025-06-27 16:05:18,633 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:18] "GET /training_status HTTP/1.1" 200 -
606
+ 2025-06-27 16:05:19,592 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:19] "GET /training_status HTTP/1.1" 200 -
607
+ 2025-06-27 16:05:25,678 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:25] "GET /training_status HTTP/1.1" 200 -
608
+ 2025-06-27 16:05:26,634 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:26] "GET /training_status HTTP/1.1" 200 -
609
+ 2025-06-27 16:05:32,711 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:32] "GET /training_status HTTP/1.1" 200 -
610
+ 2025-06-27 16:05:33,657 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:33] "GET /training_status HTTP/1.1" 200 -
611
+ 2025-06-27 16:05:39,748 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:39] "GET /training_status HTTP/1.1" 200 -
612
+ 2025-06-27 16:05:40,697 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:40] "GET /training_status HTTP/1.1" 200 -
613
+ 2025-06-27 16:05:46,790 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:46] "GET /training_status HTTP/1.1" 200 -
614
+ 2025-06-27 16:05:47,739 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:47] "GET /training_status HTTP/1.1" 200 -
615
+ 2025-06-27 16:05:53,837 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:53] "GET /training_status HTTP/1.1" 200 -
616
+ 2025-06-27 16:05:54,802 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:05:54] "GET /training_status HTTP/1.1" 200 -
617
+ 2025-06-27 16:06:00,888 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:00] "GET /training_status HTTP/1.1" 200 -
618
+ 2025-06-27 16:06:01,849 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:01] "GET /training_status HTTP/1.1" 200 -
619
+ 2025-06-27 16:06:07,916 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:07] "GET /training_status HTTP/1.1" 200 -
620
+ 2025-06-27 16:06:08,876 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:08] "GET /training_status HTTP/1.1" 200 -
621
+ 2025-06-27 16:06:14,941 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:14] "GET /training_status HTTP/1.1" 200 -
622
+ 2025-06-27 16:06:15,908 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:15] "GET /training_status HTTP/1.1" 200 -
623
+ 2025-06-27 16:06:21,995 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:21] "GET /training_status HTTP/1.1" 200 -
624
+ 2025-06-27 16:06:22,961 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:22] "GET /training_status HTTP/1.1" 200 -
625
+ 2025-06-27 16:06:29,042 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:29] "GET /training_status HTTP/1.1" 200 -
626
+ 2025-06-27 16:06:30,007 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:30] "GET /training_status HTTP/1.1" 200 -
627
+ 2025-06-27 16:06:36,074 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:36] "GET /training_status HTTP/1.1" 200 -
628
+ 2025-06-27 16:06:37,052 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:37] "GET /training_status HTTP/1.1" 200 -
629
+ 2025-06-27 16:06:43,117 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:43] "GET /training_status HTTP/1.1" 200 -
630
+ 2025-06-27 16:06:44,109 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:44] "GET /training_status HTTP/1.1" 200 -
631
+ 2025-06-27 16:06:50,174 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:50] "GET /training_status HTTP/1.1" 200 -
632
+ 2025-06-27 16:06:51,182 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:51] "GET /training_status HTTP/1.1" 200 -
633
+ 2025-06-27 16:06:57,223 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:57] "GET /training_status HTTP/1.1" 200 -
634
+ 2025-06-27 16:06:58,238 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:06:58] "GET /training_status HTTP/1.1" 200 -
635
+ 2025-06-27 16:07:04,259 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:04] "GET /training_status HTTP/1.1" 200 -
636
+ 2025-06-27 16:07:05,271 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:05] "GET /training_status HTTP/1.1" 200 -
637
+ 2025-06-27 16:07:11,301 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:11] "GET /training_status HTTP/1.1" 200 -
638
+ 2025-06-27 16:07:12,324 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:12] "GET /training_status HTTP/1.1" 200 -
639
+ 2025-06-27 16:07:18,360 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:18] "GET /training_status HTTP/1.1" 200 -
640
+ 2025-06-27 16:07:19,369 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:19] "GET /training_status HTTP/1.1" 200 -
641
+ 2025-06-27 16:07:25,392 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:25] "GET /training_status HTTP/1.1" 200 -
642
+ 2025-06-27 16:07:26,400 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:26] "GET /training_status HTTP/1.1" 200 -
643
+ 2025-06-27 16:07:32,441 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:32] "GET /training_status HTTP/1.1" 200 -
644
+ 2025-06-27 16:07:33,443 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:33] "GET /training_status HTTP/1.1" 200 -
645
+ 2025-06-27 16:07:39,485 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:39] "GET /training_status HTTP/1.1" 200 -
646
+ 2025-06-27 16:07:40,480 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:40] "GET /training_status HTTP/1.1" 200 -
647
+ 2025-06-27 16:07:46,535 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:46] "GET /training_status HTTP/1.1" 200 -
648
+ 2025-06-27 16:07:47,533 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:47] "GET /training_status HTTP/1.1" 200 -
649
+ 2025-06-27 16:07:53,572 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:53] "GET /training_status HTTP/1.1" 200 -
650
+ 2025-06-27 16:07:54,564 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:07:54] "GET /training_status HTTP/1.1" 200 -
651
+ 2025-06-27 16:08:00,617 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:00] "GET /training_status HTTP/1.1" 200 -
652
+ 2025-06-27 16:08:01,616 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:01] "GET /training_status HTTP/1.1" 200 -
653
+ 2025-06-27 16:08:07,666 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:07] "GET /training_status HTTP/1.1" 200 -
654
+ 2025-06-27 16:08:08,675 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:08] "GET /training_status HTTP/1.1" 200 -
655
+ 2025-06-27 16:08:14,702 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:14] "GET /training_status HTTP/1.1" 200 -
656
+ 2025-06-27 16:08:15,725 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:15] "GET /training_status HTTP/1.1" 200 -
657
+ 2025-06-27 16:08:21,751 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:21] "GET /training_status HTTP/1.1" 200 -
658
+ 2025-06-27 16:08:22,763 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:22] "GET /training_status HTTP/1.1" 200 -
659
+ 2025-06-27 16:08:28,802 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:28] "GET /training_status HTTP/1.1" 200 -
660
+ 2025-06-27 16:08:29,805 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:29] "GET /training_status HTTP/1.1" 200 -
661
+ 2025-06-27 16:08:35,854 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:35] "GET /training_status HTTP/1.1" 200 -
662
+ 2025-06-27 16:08:36,865 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:36] "GET /training_status HTTP/1.1" 200 -
663
+ 2025-06-27 16:08:42,919 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:42] "GET /training_status HTTP/1.1" 200 -
664
+ 2025-06-27 16:08:43,906 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:43] "GET /training_status HTTP/1.1" 200 -
665
+ 2025-06-27 16:08:49,974 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:49] "GET /training_status HTTP/1.1" 200 -
666
+ 2025-06-27 16:08:50,969 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:50] "GET /training_status HTTP/1.1" 200 -
667
+ 2025-06-27 16:08:57,026 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:57] "GET /training_status HTTP/1.1" 200 -
668
+ 2025-06-27 16:08:58,013 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:08:58] "GET /training_status HTTP/1.1" 200 -
669
+ 2025-06-27 16:09:04,098 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:04] "GET /training_status HTTP/1.1" 200 -
670
+ 2025-06-27 16:09:05,065 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:05] "GET /training_status HTTP/1.1" 200 -
671
+ 2025-06-27 16:09:11,146 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:11] "GET /training_status HTTP/1.1" 200 -
672
+ 2025-06-27 16:09:12,095 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:12] "GET /training_status HTTP/1.1" 200 -
673
+ 2025-06-27 16:09:18,193 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:18] "GET /training_status HTTP/1.1" 200 -
674
+ 2025-06-27 16:09:19,118 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:19] "GET /training_status HTTP/1.1" 200 -
675
+ 2025-06-27 16:09:25,217 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:25] "GET /training_status HTTP/1.1" 200 -
676
+ 2025-06-27 16:09:26,152 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:26] "GET /training_status HTTP/1.1" 200 -
677
+ 2025-06-27 16:09:32,252 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:32] "GET /training_status HTTP/1.1" 200 -
678
+ 2025-06-27 16:09:33,210 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:33] "GET /training_status HTTP/1.1" 200 -
679
+ 2025-06-27 16:09:39,301 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:39] "GET /training_status HTTP/1.1" 200 -
680
+ 2025-06-27 16:09:40,253 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:40] "GET /training_status HTTP/1.1" 200 -
681
+ 2025-06-27 16:09:46,344 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:46] "GET /training_status HTTP/1.1" 200 -
682
+ 2025-06-27 16:09:47,309 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:47] "GET /training_status HTTP/1.1" 200 -
683
+ 2025-06-27 16:09:53,382 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:53] "GET /training_status HTTP/1.1" 200 -
684
+ 2025-06-27 16:09:54,342 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:09:54] "GET /training_status HTTP/1.1" 200 -
685
+ 2025-06-27 16:10:00,438 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:00] "GET /training_status HTTP/1.1" 200 -
686
+ 2025-06-27 16:10:01,383 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:01] "GET /training_status HTTP/1.1" 200 -
687
+ 2025-06-27 16:10:07,489 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:07] "GET /training_status HTTP/1.1" 200 -
688
+ 2025-06-27 16:10:08,438 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:08] "GET /training_status HTTP/1.1" 200 -
689
+ 2025-06-27 16:10:14,531 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:14] "GET /training_status HTTP/1.1" 200 -
690
+ 2025-06-27 16:10:15,492 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:15] "GET /training_status HTTP/1.1" 200 -
691
+ 2025-06-27 16:10:21,584 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:21] "GET /training_status HTTP/1.1" 200 -
692
+ 2025-06-27 16:10:22,546 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:22] "GET /training_status HTTP/1.1" 200 -
693
+ 2025-06-27 16:10:28,633 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:28] "GET /training_status HTTP/1.1" 200 -
694
+ 2025-06-27 16:10:29,597 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:29] "GET /training_status HTTP/1.1" 200 -
695
+ 2025-06-27 16:10:35,667 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:35] "GET /training_status HTTP/1.1" 200 -
696
+ 2025-06-27 16:10:36,626 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:36] "GET /training_status HTTP/1.1" 200 -
697
+ 2025-06-27 16:10:42,729 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:42] "GET /training_status HTTP/1.1" 200 -
698
+ 2025-06-27 16:10:43,670 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:43] "GET /training_status HTTP/1.1" 200 -
699
+ 2025-06-27 16:10:49,780 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:49] "GET /training_status HTTP/1.1" 200 -
700
+ 2025-06-27 16:10:50,723 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:50] "GET /training_status HTTP/1.1" 200 -
701
+ 2025-06-27 16:10:56,827 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:56] "GET /training_status HTTP/1.1" 200 -
702
+ 2025-06-27 16:10:57,764 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:10:57] "GET /training_status HTTP/1.1" 200 -
703
+ 2025-06-27 16:11:03,857 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:03] "GET /training_status HTTP/1.1" 200 -
704
+ 2025-06-27 16:11:04,805 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:04] "GET /training_status HTTP/1.1" 200 -
705
+ 2025-06-27 16:11:10,908 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:10] "GET /training_status HTTP/1.1" 200 -
706
+ 2025-06-27 16:11:11,843 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:11] "GET /training_status HTTP/1.1" 200 -
707
+ 2025-06-27 16:11:17,944 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:17] "GET /training_status HTTP/1.1" 200 -
708
+ 2025-06-27 16:11:18,876 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:18] "GET /training_status HTTP/1.1" 200 -
709
+ 2025-06-27 16:11:24,991 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:24] "GET /training_status HTTP/1.1" 200 -
710
+ 2025-06-27 16:11:25,933 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:25] "GET /training_status HTTP/1.1" 200 -
711
+ 2025-06-27 16:11:32,031 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:32] "GET /training_status HTTP/1.1" 200 -
712
+ 2025-06-27 16:11:32,963 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:32] "GET /training_status HTTP/1.1" 200 -
713
+ 2025-06-27 16:11:39,081 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:39] "GET /training_status HTTP/1.1" 200 -
714
+ 2025-06-27 16:11:40,013 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:40] "GET /training_status HTTP/1.1" 200 -
715
+ 2025-06-27 16:11:46,132 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:46] "GET /training_status HTTP/1.1" 200 -
716
+ 2025-06-27 16:11:47,061 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:47] "GET /training_status HTTP/1.1" 200 -
717
+ 2025-06-27 16:11:53,192 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:53] "GET /training_status HTTP/1.1" 200 -
718
+ 2025-06-27 16:11:54,113 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:11:54] "GET /training_status HTTP/1.1" 200 -
719
+ 2025-06-27 16:12:00,235 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:00] "GET /training_status HTTP/1.1" 200 -
720
+ 2025-06-27 16:12:01,169 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:01] "GET /training_status HTTP/1.1" 200 -
721
+ 2025-06-27 16:12:07,289 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:07] "GET /training_status HTTP/1.1" 200 -
722
+ 2025-06-27 16:12:08,220 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:08] "GET /training_status HTTP/1.1" 200 -
723
+ 2025-06-27 16:12:14,347 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:14] "GET /training_status HTTP/1.1" 200 -
724
+ 2025-06-27 16:12:15,263 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:15] "GET /training_status HTTP/1.1" 200 -
725
+ 2025-06-27 16:12:21,392 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:21] "GET /training_status HTTP/1.1" 200 -
726
+ 2025-06-27 16:12:22,297 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:22] "GET /training_status HTTP/1.1" 200 -
727
+ 2025-06-27 16:12:28,429 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:28] "GET /training_status HTTP/1.1" 200 -
728
+ 2025-06-27 16:12:29,337 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:29] "GET /training_status HTTP/1.1" 200 -
729
+ 2025-06-27 16:12:35,468 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:35] "GET /training_status HTTP/1.1" 200 -
730
+ 2025-06-27 16:12:36,377 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:36] "GET /training_status HTTP/1.1" 200 -
731
+ 2025-06-27 16:12:42,505 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:42] "GET /training_status HTTP/1.1" 200 -
732
+ 2025-06-27 16:12:43,424 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:43] "GET /training_status HTTP/1.1" 200 -
733
+ 2025-06-27 16:12:49,557 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:49] "GET /training_status HTTP/1.1" 200 -
734
+ 2025-06-27 16:12:50,463 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:50] "GET /training_status HTTP/1.1" 200 -
735
+ 2025-06-27 16:12:56,595 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:56] "GET /training_status HTTP/1.1" 200 -
736
+ 2025-06-27 16:12:57,511 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:12:57] "GET /training_status HTTP/1.1" 200 -
737
+ 2025-06-27 16:13:03,642 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:03] "GET /training_status HTTP/1.1" 200 -
738
+ 2025-06-27 16:13:04,567 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:04] "GET /training_status HTTP/1.1" 200 -
739
+ 2025-06-27 16:13:10,690 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:10] "GET /training_status HTTP/1.1" 200 -
740
+ 2025-06-27 16:13:11,619 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:11] "GET /training_status HTTP/1.1" 200 -
741
+ 2025-06-27 16:13:17,740 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:17] "GET /training_status HTTP/1.1" 200 -
742
+ 2025-06-27 16:13:18,677 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:18] "GET /training_status HTTP/1.1" 200 -
743
+ 2025-06-27 16:13:24,787 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:24] "GET /training_status HTTP/1.1" 200 -
744
+ 2025-06-27 16:13:25,731 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:25] "GET /training_status HTTP/1.1" 200 -
745
+ 2025-06-27 16:13:31,839 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:31] "GET /training_status HTTP/1.1" 200 -
746
+ 2025-06-27 16:13:32,768 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:32] "GET /training_status HTTP/1.1" 200 -
747
+ 2025-06-27 16:13:38,884 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:38] "GET /training_status HTTP/1.1" 200 -
748
+ 2025-06-27 16:13:39,812 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:39] "GET /training_status HTTP/1.1" 200 -
749
+ 2025-06-27 16:13:45,924 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:45] "GET /training_status HTTP/1.1" 200 -
750
+ 2025-06-27 16:13:46,857 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:46] "GET /training_status HTTP/1.1" 200 -
751
+ 2025-06-27 16:13:53,002 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:53] "GET /training_status HTTP/1.1" 200 -
752
+ 2025-06-27 16:13:53,901 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:13:53] "GET /training_status HTTP/1.1" 200 -
753
+ 2025-06-27 16:14:00,041 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:00] "GET /training_status HTTP/1.1" 200 -
754
+ 2025-06-27 16:14:00,938 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:00] "GET /training_status HTTP/1.1" 200 -
755
+ 2025-06-27 16:14:07,077 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:07] "GET /training_status HTTP/1.1" 200 -
756
+ 2025-06-27 16:14:08,005 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:08] "GET /training_status HTTP/1.1" 200 -
757
+ 2025-06-27 16:14:14,117 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:14] "GET /training_status HTTP/1.1" 200 -
758
+ 2025-06-27 16:14:15,034 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:15] "GET /training_status HTTP/1.1" 200 -
759
+ 2025-06-27 16:14:21,153 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:21] "GET /training_status HTTP/1.1" 200 -
760
+ 2025-06-27 16:14:22,086 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:22] "GET /training_status HTTP/1.1" 200 -
761
+ 2025-06-27 16:14:28,213 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:28] "GET /training_status HTTP/1.1" 200 -
762
+ 2025-06-27 16:14:29,135 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:29] "GET /training_status HTTP/1.1" 200 -
763
+ 2025-06-27 16:14:35,261 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:35] "GET /training_status HTTP/1.1" 200 -
764
+ 2025-06-27 16:14:36,180 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:36] "GET /training_status HTTP/1.1" 200 -
765
+ 2025-06-27 16:14:42,302 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:42] "GET /training_status HTTP/1.1" 200 -
766
+ 2025-06-27 16:14:43,232 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:43] "GET /training_status HTTP/1.1" 200 -
767
+ 2025-06-27 16:14:49,356 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:49] "GET /training_status HTTP/1.1" 200 -
768
+ 2025-06-27 16:14:50,287 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:50] "GET /training_status HTTP/1.1" 200 -
769
+ 2025-06-27 16:14:56,427 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:56] "GET /training_status HTTP/1.1" 200 -
770
+ 2025-06-27 16:14:57,344 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:14:57] "GET /training_status HTTP/1.1" 200 -
771
+ 2025-06-27 16:15:03,478 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:03] "GET /training_status HTTP/1.1" 200 -
772
+ 2025-06-27 16:15:04,395 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:04] "GET /training_status HTTP/1.1" 200 -
773
+ 2025-06-27 16:15:10,524 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:10] "GET /training_status HTTP/1.1" 200 -
774
+ 2025-06-27 16:15:11,435 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:11] "GET /training_status HTTP/1.1" 200 -
775
+ 2025-06-27 16:15:17,564 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:17] "GET /training_status HTTP/1.1" 200 -
776
+ 2025-06-27 16:15:18,470 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:18] "GET /training_status HTTP/1.1" 200 -
777
+ 2025-06-27 16:15:24,609 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:24] "GET /training_status HTTP/1.1" 200 -
778
+ 2025-06-27 16:15:25,525 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:25] "GET /training_status HTTP/1.1" 200 -
779
+ 2025-06-27 16:15:31,671 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:31] "GET /training_status HTTP/1.1" 200 -
780
+ 2025-06-27 16:15:32,588 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:32] "GET /training_status HTTP/1.1" 200 -
781
+ 2025-06-27 16:15:38,713 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:38] "GET /training_status HTTP/1.1" 200 -
782
+ 2025-06-27 16:15:39,630 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:39] "GET /training_status HTTP/1.1" 200 -
783
+ 2025-06-27 16:15:45,741 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:45] "GET /training_status HTTP/1.1" 200 -
784
+ 2025-06-27 16:15:46,674 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:46] "GET /training_status HTTP/1.1" 200 -
785
+ 2025-06-27 16:15:52,803 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:52] "GET /training_status HTTP/1.1" 200 -
786
+ 2025-06-27 16:15:53,732 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:53] "GET /training_status HTTP/1.1" 200 -
787
+ 2025-06-27 16:15:59,843 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:15:59] "GET /training_status HTTP/1.1" 200 -
788
+ 2025-06-27 16:16:00,780 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:00] "GET /training_status HTTP/1.1" 200 -
789
+ 2025-06-27 16:16:06,900 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:06] "GET /training_status HTTP/1.1" 200 -
790
+ 2025-06-27 16:16:07,832 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:07] "GET /training_status HTTP/1.1" 200 -
791
+ 2025-06-27 16:16:13,942 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:13] "GET /training_status HTTP/1.1" 200 -
792
+ 2025-06-27 16:16:14,878 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:14] "GET /training_status HTTP/1.1" 200 -
793
+ 2025-06-27 16:16:21,009 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:21] "GET /training_status HTTP/1.1" 200 -
794
+ 2025-06-27 16:16:21,925 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:21] "GET /training_status HTTP/1.1" 200 -
795
+ 2025-06-27 16:16:28,045 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:28] "GET /training_status HTTP/1.1" 200 -
796
+ 2025-06-27 16:16:28,962 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:28] "GET /training_status HTTP/1.1" 200 -
797
+ 2025-06-27 16:16:35,075 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:35] "GET /training_status HTTP/1.1" 200 -
798
+ 2025-06-27 16:16:36,014 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:36] "GET /training_status HTTP/1.1" 200 -
799
+ 2025-06-27 16:16:42,104 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:42] "GET /training_status HTTP/1.1" 200 -
800
+ 2025-06-27 16:16:43,058 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:43] "GET /training_status HTTP/1.1" 200 -
801
+ 2025-06-27 16:16:49,142 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:49] "GET /training_status HTTP/1.1" 200 -
802
+ 2025-06-27 16:16:50,095 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:50] "GET /training_status HTTP/1.1" 200 -
803
+ 2025-06-27 16:16:56,178 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:56] "GET /training_status HTTP/1.1" 200 -
804
+ 2025-06-27 16:16:57,136 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:16:57] "GET /training_status HTTP/1.1" 200 -
805
+ 2025-06-27 16:17:03,227 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:03] "GET /training_status HTTP/1.1" 200 -
806
+ 2025-06-27 16:17:04,169 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:04] "GET /training_status HTTP/1.1" 200 -
807
+ 2025-06-27 16:17:10,258 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:10] "GET /training_status HTTP/1.1" 200 -
808
+ 2025-06-27 16:17:11,204 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:11] "GET /training_status HTTP/1.1" 200 -
809
+ 2025-06-27 16:17:17,306 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:17] "GET /training_status HTTP/1.1" 200 -
810
+ 2025-06-27 16:17:18,257 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:18] "GET /training_status HTTP/1.1" 200 -
811
+ 2025-06-27 16:17:24,350 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:24] "GET /training_status HTTP/1.1" 200 -
812
+ 2025-06-27 16:17:25,307 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:25] "GET /training_status HTTP/1.1" 200 -
813
+ 2025-06-27 16:17:31,404 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:31] "GET /training_status HTTP/1.1" 200 -
814
+ 2025-06-27 16:17:32,361 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:32] "GET /training_status HTTP/1.1" 200 -
815
+ 2025-06-27 16:17:38,459 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:38] "GET /training_status HTTP/1.1" 200 -
816
+ 2025-06-27 16:17:39,405 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:39] "GET /training_status HTTP/1.1" 200 -
817
+ 2025-06-27 16:17:45,496 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:45] "GET /training_status HTTP/1.1" 200 -
818
+ 2025-06-27 16:17:46,454 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:46] "GET /training_status HTTP/1.1" 200 -
819
+ 2025-06-27 16:17:52,558 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:52] "GET /training_status HTTP/1.1" 200 -
820
+ 2025-06-27 16:17:53,507 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:53] "GET /training_status HTTP/1.1" 200 -
821
+ 2025-06-27 16:17:59,618 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:17:59] "GET /training_status HTTP/1.1" 200 -
822
+ 2025-06-27 16:18:00,566 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:00] "GET /training_status HTTP/1.1" 200 -
823
+ 2025-06-27 16:18:06,651 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:06] "GET /training_status HTTP/1.1" 200 -
824
+ 2025-06-27 16:18:07,610 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:07] "GET /training_status HTTP/1.1" 200 -
825
+ 2025-06-27 16:18:13,686 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:13] "GET /training_status HTTP/1.1" 200 -
826
+ 2025-06-27 16:18:14,646 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:14] "GET /training_status HTTP/1.1" 200 -
827
+ 2025-06-27 16:18:20,735 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:20] "GET /training_status HTTP/1.1" 200 -
828
+ 2025-06-27 16:18:21,683 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:21] "GET /training_status HTTP/1.1" 200 -
829
+ 2025-06-27 16:18:27,767 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:27] "GET /training_status HTTP/1.1" 200 -
830
+ 2025-06-27 16:18:28,720 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:28] "GET /training_status HTTP/1.1" 200 -
831
+ 2025-06-27 16:18:34,828 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:34] "GET /training_status HTTP/1.1" 200 -
832
+ 2025-06-27 16:18:35,750 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:35] "GET /training_status HTTP/1.1" 200 -
833
+ 2025-06-27 16:18:41,851 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:41] "GET /training_status HTTP/1.1" 200 -
834
+ 2025-06-27 16:18:42,791 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:42] "GET /training_status HTTP/1.1" 200 -
835
+ 2025-06-27 16:18:48,887 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:48] "GET /training_status HTTP/1.1" 200 -
836
+ 2025-06-27 16:18:49,824 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:49] "GET /training_status HTTP/1.1" 200 -
837
+ 2025-06-27 16:18:55,939 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:55] "GET /training_status HTTP/1.1" 200 -
838
+ 2025-06-27 16:18:56,869 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:18:56] "GET /training_status HTTP/1.1" 200 -
839
+ 2025-06-27 16:19:02,987 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:02] "GET /training_status HTTP/1.1" 200 -
840
+ 2025-06-27 16:19:03,917 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:03] "GET /training_status HTTP/1.1" 200 -
841
+ 2025-06-27 16:19:10,036 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:10] "GET /training_status HTTP/1.1" 200 -
842
+ 2025-06-27 16:19:10,963 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:10] "GET /training_status HTTP/1.1" 200 -
843
+ 2025-06-27 16:19:17,093 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:17] "GET /training_status HTTP/1.1" 200 -
844
+ 2025-06-27 16:19:17,998 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:17] "GET /training_status HTTP/1.1" 200 -
845
+ 2025-06-27 16:19:24,143 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:24] "GET /training_status HTTP/1.1" 200 -
846
+ 2025-06-27 16:19:25,043 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:25] "GET /training_status HTTP/1.1" 200 -
847
+ 2025-06-27 16:19:31,184 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:31] "GET /training_status HTTP/1.1" 200 -
848
+ 2025-06-27 16:19:32,104 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:32] "GET /training_status HTTP/1.1" 200 -
849
+ 2025-06-27 16:19:38,237 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:38] "GET /training_status HTTP/1.1" 200 -
850
+ 2025-06-27 16:19:39,155 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:39] "GET /training_status HTTP/1.1" 200 -
851
+ 2025-06-27 16:19:45,276 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:45] "GET /training_status HTTP/1.1" 200 -
852
+ 2025-06-27 16:19:46,197 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:46] "GET /training_status HTTP/1.1" 200 -
853
+ 2025-06-27 16:19:52,305 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:52] "GET /training_status HTTP/1.1" 200 -
854
+ 2025-06-27 16:19:53,219 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:53] "GET /training_status HTTP/1.1" 200 -
855
+ 2025-06-27 16:19:59,338 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:19:59] "GET /training_status HTTP/1.1" 200 -
856
+ 2025-06-27 16:20:00,255 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:00] "GET /training_status HTTP/1.1" 200 -
857
+ 2025-06-27 16:20:06,385 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:06] "GET /training_status HTTP/1.1" 200 -
858
+ 2025-06-27 16:20:07,283 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:07] "GET /training_status HTTP/1.1" 200 -
859
+ 2025-06-27 16:20:13,426 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:13] "GET /training_status HTTP/1.1" 200 -
860
+ 2025-06-27 16:20:14,339 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:14] "GET /training_status HTTP/1.1" 200 -
861
+ 2025-06-27 16:20:20,488 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:20] "GET /training_status HTTP/1.1" 200 -
862
+ 2025-06-27 16:20:21,389 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:21] "GET /training_status HTTP/1.1" 200 -
863
+ 2025-06-27 16:20:27,536 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:27] "GET /training_status HTTP/1.1" 200 -
864
+ 2025-06-27 16:20:28,435 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:28] "GET /training_status HTTP/1.1" 200 -
865
+ 2025-06-27 16:20:34,568 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:34] "GET /training_status HTTP/1.1" 200 -
866
+ 2025-06-27 16:20:35,463 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:35] "GET /training_status HTTP/1.1" 200 -
867
+ 2025-06-27 16:20:41,617 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:41] "GET /training_status HTTP/1.1" 200 -
868
+ 2025-06-27 16:20:42,507 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:42] "GET /training_status HTTP/1.1" 200 -
869
+ 2025-06-27 16:20:48,656 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:48] "GET /training_status HTTP/1.1" 200 -
870
+ 2025-06-27 16:20:49,573 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:49] "GET /training_status HTTP/1.1" 200 -
871
+ 2025-06-27 16:20:55,713 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:55] "GET /training_status HTTP/1.1" 200 -
872
+ 2025-06-27 16:20:56,623 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:20:56] "GET /training_status HTTP/1.1" 200 -
873
+ 2025-06-27 16:21:02,763 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:02] "GET /training_status HTTP/1.1" 200 -
874
+ 2025-06-27 16:21:03,682 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:03] "GET /training_status HTTP/1.1" 200 -
875
+ 2025-06-27 16:21:09,806 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:09] "GET /training_status HTTP/1.1" 200 -
876
+ 2025-06-27 16:21:10,739 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:10] "GET /training_status HTTP/1.1" 200 -
877
+ 2025-06-27 16:21:16,835 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:16] "GET /training_status HTTP/1.1" 200 -
878
+ 2025-06-27 16:21:17,761 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:17] "GET /training_status HTTP/1.1" 200 -
879
+ 2025-06-27 16:21:23,872 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:23] "GET /training_status HTTP/1.1" 200 -
880
+ 2025-06-27 16:21:24,793 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:24] "GET /training_status HTTP/1.1" 200 -
881
+ 2025-06-27 16:21:30,924 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:30] "GET /training_status HTTP/1.1" 200 -
882
+ 2025-06-27 16:21:31,845 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:31] "GET /training_status HTTP/1.1" 200 -
883
+ 2025-06-27 16:21:37,972 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:37] "GET /training_status HTTP/1.1" 200 -
884
+ 2025-06-27 16:21:38,871 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:38] "GET /training_status HTTP/1.1" 200 -
885
+ 2025-06-27 16:21:45,019 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:45] "GET /training_status HTTP/1.1" 200 -
886
+ 2025-06-27 16:21:45,920 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:45] "GET /training_status HTTP/1.1" 200 -
887
+ 2025-06-27 16:21:52,056 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:52] "GET /training_status HTTP/1.1" 200 -
888
+ 2025-06-27 16:21:52,960 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:52] "GET /training_status HTTP/1.1" 200 -
889
+ 2025-06-27 16:21:59,094 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:59] "GET /training_status HTTP/1.1" 200 -
890
+ 2025-06-27 16:21:59,992 - werkzeug - INFO - 127.0.0.1 - - [27/Jun/2025 16:21:59] "GET /training_status HTTP/1.1" 200 -
notebooks/data_exploration.ipynb ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [],
3
+ "metadata": {},
4
+ "nbformat": 4,
5
+ "nbformat_minor": 4
6
+ }
notebooks/model_evaluation.ipynb ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [],
3
+ "metadata": {},
4
+ "nbformat": 4,
5
+ "nbformat_minor": 4
6
+ }
requirements.txt CHANGED
@@ -18,9 +18,13 @@ tensorflow-privacy
18
  pysyft
19
 
20
  # API and web
21
- flask
22
  fastapi
23
  uvicorn
 
 
 
 
24
 
25
  # Testing and development
26
  pytest
 
18
  pysyft
19
 
20
  # API and web
21
+ flask>=2.0.0
22
  fastapi
23
  uvicorn
24
+ requests>=2.25.0
25
+
26
+ # Configuration and utilities
27
+ pyyaml>=5.4.0
28
 
29
  # Testing and development
30
  pytest
run_two_clients.sh ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # Script to run two clients with different configs
3
+
4
+ # Start first client
5
+ python -m src.main --mode client --config config/client_config.yaml &
6
+
7
+ # Start second client
8
+ python -m src.main --mode client --config config/client_config_2.yaml &
9
+
10
+ # Wait for both clients to finish (optional)
11
+ wait
src/api/__init__.py ADDED
File without changes
src/api/client.py ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ HTTP Client for Federated Learning
3
+ Handles communication with the federated server
4
+ """
5
+
6
+ import requests
7
+ import json
8
+ import logging
9
+ import time
10
+ from typing import Dict, Any, Optional, List
11
+
12
+ logger = logging.getLogger(__name__)
13
+
14
+ class FederatedHTTPClient:
15
+ def __init__(self, server_url: str, client_id: str, timeout: int = 30):
16
+ self.server_url = server_url.rstrip('/')
17
+ self.client_id = client_id
18
+ self.timeout = timeout
19
+ self.session = requests.Session()
20
+
21
+ def register(self, client_info: Dict[str, Any] = None) -> Dict[str, Any]:
22
+ """Register this client with the server"""
23
+ try:
24
+ payload = {
25
+ 'client_id': self.client_id,
26
+ 'client_info': client_info or {}
27
+ }
28
+
29
+ response = self.session.post(
30
+ f"{self.server_url}/register",
31
+ json=payload,
32
+ timeout=self.timeout
33
+ )
34
+ response.raise_for_status()
35
+
36
+ result = response.json()
37
+ logger.info(f"Client {self.client_id} registered successfully")
38
+ return result
39
+
40
+ except requests.exceptions.RequestException as e:
41
+ logger.error(f"Failed to register client {self.client_id}: {str(e)}")
42
+ raise
43
+
44
+ def get_global_model(self) -> Dict[str, Any]:
45
+ """Get the current global model from server"""
46
+ try:
47
+ payload = {'client_id': self.client_id}
48
+
49
+ response = self.session.post(
50
+ f"{self.server_url}/get_model",
51
+ json=payload,
52
+ timeout=self.timeout
53
+ )
54
+ response.raise_for_status()
55
+
56
+ result = response.json()
57
+ logger.debug(f"Retrieved global model for round {result.get('round', 'unknown')}")
58
+ return result
59
+
60
+ except requests.exceptions.RequestException as e:
61
+ logger.error(f"Failed to get global model: {str(e)}")
62
+ raise
63
+
64
+ def submit_model_update(self, model_weights: List, metrics: Dict[str, Any] = None) -> Dict[str, Any]:
65
+ """Submit model update to server"""
66
+ try:
67
+ payload = {
68
+ 'client_id': self.client_id,
69
+ 'model_weights': model_weights,
70
+ 'metrics': metrics or {}
71
+ }
72
+
73
+ response = self.session.post(
74
+ f"{self.server_url}/submit_update",
75
+ json=payload,
76
+ timeout=self.timeout
77
+ )
78
+ response.raise_for_status()
79
+
80
+ result = response.json()
81
+ logger.info(f"Model update submitted successfully by client {self.client_id}")
82
+ return result
83
+
84
+ except requests.exceptions.RequestException as e:
85
+ logger.error(f"Failed to submit model update: {str(e)}")
86
+ raise
87
+
88
+ def get_training_status(self) -> Dict[str, Any]:
89
+ """Get current training status from server"""
90
+ try:
91
+ response = self.session.get(
92
+ f"{self.server_url}/training_status",
93
+ timeout=self.timeout
94
+ )
95
+ response.raise_for_status()
96
+
97
+ return response.json()
98
+
99
+ except requests.exceptions.RequestException as e:
100
+ logger.error(f"Failed to get training status: {str(e)}")
101
+ raise
102
+
103
+ def health_check(self) -> bool:
104
+ """Check if server is healthy"""
105
+ try:
106
+ response = self.session.get(
107
+ f"{self.server_url}/health",
108
+ timeout=5 # Short timeout for health checks
109
+ )
110
+ response.raise_for_status()
111
+
112
+ result = response.json()
113
+ return result.get('status') == 'healthy'
114
+
115
+ except requests.exceptions.RequestException:
116
+ return False
117
+
118
+ def wait_for_server(self, max_wait: int = 60, check_interval: int = 5) -> bool:
119
+ """Wait for server to become available"""
120
+ start_time = time.time()
121
+
122
+ while time.time() - start_time < max_wait:
123
+ if self.health_check():
124
+ logger.info(f"Server is available at {self.server_url}")
125
+ return True
126
+
127
+ logger.info(f"Waiting for server at {self.server_url}...")
128
+ time.sleep(check_interval)
129
+
130
+ logger.error(f"Server not available after {max_wait} seconds")
131
+ return False
132
+
133
+ def rag_query(self, query: str) -> Dict[str, Any]:
134
+ """Submit a RAG query to the server"""
135
+ try:
136
+ payload = {
137
+ 'query': query,
138
+ 'client_id': self.client_id
139
+ }
140
+
141
+ response = self.session.post(
142
+ f"{self.server_url}/rag/query",
143
+ json=payload,
144
+ timeout=self.timeout
145
+ )
146
+ response.raise_for_status()
147
+
148
+ return response.json()
149
+
150
+ except requests.exceptions.RequestException as e:
151
+ logger.error(f"Failed to submit RAG query: {str(e)}")
152
+ raise
153
+
154
+ def close(self):
155
+ """Close the HTTP session"""
156
+ self.session.close()
src/api/server.py ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ RESTful API for Federated Learning Server
3
+ Handles client registration, model updates, and coordination
4
+ """
5
+
6
+ from flask import Flask, request, jsonify
7
+ import logging
8
+ import threading
9
+ import time
10
+ from typing import Dict, Any, List
11
+ from ..server.coordinator import FederatedCoordinator
12
+ from ..utils.metrics import calculate_model_similarity
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+ class FederatedAPI:
17
+ def __init__(self, coordinator: FederatedCoordinator, host: str = "0.0.0.0", port: int = 8080):
18
+ self.app = Flask(__name__)
19
+ self.coordinator = coordinator
20
+ self.host = host
21
+ self.port = port
22
+ self._setup_routes()
23
+
24
+ def _setup_routes(self):
25
+ """Setup API routes"""
26
+
27
+ @self.app.route('/health', methods=['GET'])
28
+ def health_check():
29
+ """Health check endpoint"""
30
+ return jsonify({
31
+ 'status': 'healthy',
32
+ 'timestamp': time.time(),
33
+ 'active_clients': len(self.coordinator.clients),
34
+ 'current_round': getattr(self.coordinator, 'current_round', 0)
35
+ })
36
+
37
+ @self.app.route('/register', methods=['POST'])
38
+ def register_client():
39
+ """Register a new client"""
40
+ try:
41
+ data = request.get_json()
42
+ client_id = data.get('client_id')
43
+ client_info = data.get('client_info', {})
44
+
45
+ if not client_id:
46
+ return jsonify({'error': 'client_id is required'}), 400
47
+
48
+ success = self.coordinator.register_client(client_id, client_info)
49
+
50
+ if success:
51
+ return jsonify({
52
+ 'status': 'registered',
53
+ 'client_id': client_id,
54
+ 'server_config': self.coordinator.get_client_config()
55
+ })
56
+ else:
57
+ return jsonify({'error': 'Registration failed'}), 400
58
+
59
+ except Exception as e:
60
+ logger.error(f"Error registering client: {str(e)}")
61
+ return jsonify({'error': str(e)}), 500
62
+
63
+ @self.app.route('/get_model', methods=['POST'])
64
+ def get_global_model():
65
+ """Get the current global model"""
66
+ try:
67
+ data = request.get_json()
68
+ client_id = data.get('client_id')
69
+
70
+ if not client_id or client_id not in self.coordinator.clients:
71
+ return jsonify({'error': 'Invalid client_id'}), 400
72
+
73
+ model_weights = self.coordinator.get_global_model()
74
+
75
+ return jsonify({
76
+ 'model_weights': model_weights,
77
+ 'round': getattr(self.coordinator, 'current_round', 0),
78
+ 'timestamp': time.time()
79
+ })
80
+
81
+ except Exception as e:
82
+ logger.error(f"Error getting global model: {str(e)}")
83
+ return jsonify({'error': str(e)}), 500
84
+
85
+ @self.app.route('/submit_update', methods=['POST'])
86
+ def submit_model_update():
87
+ """Submit a model update from client"""
88
+ try:
89
+ data = request.get_json()
90
+ client_id = data.get('client_id')
91
+ model_weights = data.get('model_weights')
92
+ training_metrics = data.get('metrics', {})
93
+
94
+ if not client_id or not model_weights:
95
+ return jsonify({'error': 'client_id and model_weights are required'}), 400
96
+
97
+ if client_id not in self.coordinator.clients:
98
+ return jsonify({'error': 'Client not registered'}), 400
99
+
100
+ # Store the update
101
+ self.coordinator.receive_model_update(client_id, model_weights, training_metrics)
102
+
103
+ return jsonify({
104
+ 'status': 'update_received',
105
+ 'client_id': client_id,
106
+ 'timestamp': time.time()
107
+ })
108
+
109
+ except Exception as e:
110
+ logger.error(f"Error submitting model update: {str(e)}")
111
+ return jsonify({'error': str(e)}), 500
112
+
113
+ @self.app.route('/training_status', methods=['GET'])
114
+ def get_training_status():
115
+ """Get current training status"""
116
+ try:
117
+ return jsonify({
118
+ 'current_round': getattr(self.coordinator, 'current_round', 0),
119
+ 'total_rounds': self.coordinator.config.get('federated', {}).get('num_rounds', 10),
120
+ 'active_clients': len(self.coordinator.clients),
121
+ 'clients_ready': len(getattr(self.coordinator, 'client_updates', {})),
122
+ 'min_clients': self.coordinator.config.get('federated', {}).get('min_clients', 2),
123
+ 'training_active': getattr(self.coordinator, 'training_active', False)
124
+ })
125
+
126
+ except Exception as e:
127
+ logger.error(f"Error getting training status: {str(e)}")
128
+ return jsonify({'error': str(e)}), 500
129
+
130
+ @self.app.route('/rag/query', methods=['POST'])
131
+ def rag_query():
132
+ """Handle RAG queries"""
133
+ try:
134
+ data = request.get_json()
135
+ query = data.get('query')
136
+ client_id = data.get('client_id')
137
+
138
+ if not query:
139
+ return jsonify({'error': 'query is required'}), 400
140
+
141
+ # This will be implemented when we integrate RAG
142
+ return jsonify({
143
+ 'response': 'RAG functionality coming soon',
144
+ 'query': query,
145
+ 'timestamp': time.time()
146
+ })
147
+
148
+ except Exception as e:
149
+ logger.error(f"Error processing RAG query: {str(e)}")
150
+ return jsonify({'error': str(e)}), 500
151
+
152
+ def run(self, debug: bool = False):
153
+ """Run the API server"""
154
+ logger.info(f"Starting Federated API server on {self.host}:{self.port}")
155
+ self.app.run(host=self.host, port=self.port, debug=debug, threaded=True)
156
+
157
+ def run_threaded(self, debug: bool = False):
158
+ """Run the API server in a separate thread"""
159
+ def run_server():
160
+ self.app.run(host=self.host, port=self.port, debug=debug, threaded=True)
161
+
162
+ thread = threading.Thread(target=run_server, daemon=True)
163
+ thread.start()
164
+ logger.info(f"Federated API server started in background on {self.host}:{self.port}")
165
+ return thread
src/client/model.py CHANGED
@@ -1,69 +1,156 @@
1
  """model.py module."""
2
 
3
- from typing import Dict, List
4
  import tensorflow as tf
5
  import numpy as np
6
  import logging
 
 
 
7
 
8
  class FederatedClient:
9
- def __init__(self, client_id: int, config: Dict):
10
  """Initialize the federated client."""
11
- self.client_id = client_id
12
  self.config = config.get('client', {})
13
  self.model = self._build_model()
 
 
 
 
 
 
 
 
 
14
 
15
  def start(self):
16
- """Start the federated client process."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
  logger = logging.getLogger(__name__)
18
- logger.info(f"Client {self.client_id} started")
19
- logger.info(f"Client config: {self.config}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
 
21
  try:
22
- # Simulate some data
23
- logger.info("Generating training data...")
 
 
 
 
 
 
 
24
  X, y = self._generate_dummy_data()
25
- logger.info(f"Generated data shapes - X: {X.shape}, y: {y.shape}")
26
 
27
  # Train locally
28
- logger.info("Starting local training...")
29
  history = self.train_local((X, y))
30
 
31
- # Log training metrics
32
- losses = history.get('loss', [])
33
- logger.info("\nTraining Progress Summary:")
34
- logger.info("-" * 30)
35
- for epoch, loss in enumerate(losses, 1):
36
- logger.info(f"Epoch {epoch:2d}/{len(losses)}: loss = {loss:.4f}")
37
-
38
- final_loss = losses[-1]
39
- logger.info(f"\nTraining completed - Final loss: {final_loss:.4f}")
40
-
41
- # Log model summary in a simpler format
42
- logger.info("\nModel Architecture:")
43
- logger.info("-" * 30)
44
- logger.info("Layer (Output Shape) -> Params")
45
- total_params = 0
46
- for layer in self.model.layers:
47
- params = layer.count_params()
48
- total_params += params
49
- logger.info(f"{layer.name} {layer.output.shape} -> {params:,} params")
50
- logger.info(f"Total Parameters: {total_params:,}")
51
 
52
  except Exception as e:
53
- logger.error(f"Error during client execution: {str(e)}")
54
  raise
55
 
56
  def _generate_dummy_data(self):
57
  """Generate dummy data for testing."""
58
- num_samples = 100
59
- input_dim = 32 # Match with model's input dimension
60
-
61
- # Generate input data
62
- X = tf.random.normal((num_samples, input_dim))
63
- # Generate target data (for this example, we'll predict the sum of inputs)
64
- y = tf.reduce_sum(X, axis=1, keepdims=True)
65
-
66
- return X, y
 
 
 
 
 
67
 
68
  def _build_model(self):
69
  """Build the initial model architecture."""
@@ -87,27 +174,30 @@ class FederatedClient:
87
  logger = logging.getLogger(__name__)
88
  X, y = data
89
 
 
 
 
 
 
 
90
  # Log training parameters
91
- logger.info(f"\nTraining Parameters:")
92
- logger.info("-" * 50)
93
  logger.info(f"Input shape: {X.shape}")
94
  logger.info(f"Output shape: {y.shape}")
95
- logger.info(f"Batch size: {self.config.get('data', {}).get('batch_size', 32)}")
96
  logger.info(f"Epochs: {self.config.get('training', {}).get('local_epochs', 5)}")
97
- logger.info(f"Learning rate: {self.config.get('training', {}).get('learning_rate', 0.001)}")
98
- logger.info("-" * 50)
99
 
100
  class LogCallback(tf.keras.callbacks.Callback):
101
  def on_epoch_end(self, epoch, logs=None):
102
- logger.info(f"Epoch {epoch + 1} - loss: {logs['loss']:.4f}")
103
 
104
- # Enable verbose mode for training
105
  history = self.model.fit(
106
  X, y,
107
- batch_size=self.config.get('data', {}).get('batch_size', 32),
108
- epochs=self.config.get('training', {}).get('local_epochs', 5),
109
- verbose=0, # Disable default verbose output
110
- callbacks=[LogCallback()] # Use our custom callback
111
  )
112
  return history.history
113
 
 
1
  """model.py module."""
2
 
3
+ from typing import Dict, List, Optional, Tuple
4
  import tensorflow as tf
5
  import numpy as np
6
  import logging
7
+ import time
8
+ from ..api.client import FederatedHTTPClient
9
+ from .data_handler import FinancialDataHandler
10
 
11
  class FederatedClient:
12
+ def __init__(self, client_id: str, config: Dict, server_url: Optional[str] = None):
13
  """Initialize the federated client."""
14
+ self.client_id = str(client_id)
15
  self.config = config.get('client', {})
16
  self.model = self._build_model()
17
+ self.data_handler = FinancialDataHandler(self.config)
18
+
19
+ # HTTP client for server communication
20
+ self.server_url = server_url or self.config.get('server_url', 'http://localhost:8080')
21
+ self.http_client = FederatedHTTPClient(self.server_url, self.client_id)
22
+
23
+ # Training state
24
+ self.registered = False
25
+ self.current_round = 0
26
 
27
  def start(self):
28
+ """Start the federated client process with server communication."""
29
+ logger = logging.getLogger(__name__)
30
+ logger.info(f"Client {self.client_id} starting...")
31
+
32
+ try:
33
+ # Wait for server to be available
34
+ if not self.http_client.wait_for_server():
35
+ raise ConnectionError(f"Cannot connect to server at {self.server_url}")
36
+
37
+ # Register with server
38
+ self._register_with_server()
39
+
40
+ # Main federated learning loop
41
+ self._federated_learning_loop()
42
+
43
+ except Exception as e:
44
+ logger.error(f"Error during client execution: {str(e)}")
45
+ raise
46
+ finally:
47
+ self.http_client.close()
48
+
49
+ def _register_with_server(self):
50
+ """Register this client with the federated server"""
51
+ logger = logging.getLogger(__name__)
52
+
53
+ try:
54
+ # Generate local data to get client info
55
+ X, y = self._generate_dummy_data()
56
+
57
+ client_info = {
58
+ 'dataset_size': len(X),
59
+ 'model_params': self.model.count_params(),
60
+ 'capabilities': ['training', 'inference']
61
+ }
62
+
63
+ response = self.http_client.register(client_info)
64
+ self.registered = True
65
+
66
+ logger.info(f"Successfully registered with server")
67
+ logger.info(f"Dataset size: {client_info['dataset_size']}")
68
+ logger.info(f"Model parameters: {client_info['model_params']:,}")
69
+
70
+ except Exception as e:
71
+ logger.error(f"Failed to register with server: {str(e)}")
72
+ raise
73
+
74
+ def _federated_learning_loop(self):
75
+ """Main federated learning loop"""
76
  logger = logging.getLogger(__name__)
77
+
78
+ while True:
79
+ try:
80
+ # Get training status from server
81
+ status = self.http_client.get_training_status()
82
+
83
+ if not status.get('training_active', True):
84
+ logger.info("Training completed on server")
85
+ break
86
+
87
+ server_round = status.get('current_round', 0)
88
+
89
+ if server_round > self.current_round:
90
+ self._participate_in_round(server_round)
91
+ self.current_round = server_round
92
+
93
+ time.sleep(5) # Check every 5 seconds
94
+
95
+ except Exception as e:
96
+ logger.error(f"Error in federated learning loop: {str(e)}")
97
+ time.sleep(10) # Wait longer on error
98
+
99
+ def _participate_in_round(self, round_num: int):
100
+ """Participate in a federated learning round"""
101
+ logger = logging.getLogger(__name__)
102
+ logger.info(f"Participating in round {round_num}")
103
 
104
  try:
105
+ # Get global model from server
106
+ model_response = self.http_client.get_global_model()
107
+ global_weights = model_response.get('model_weights')
108
+
109
+ if global_weights:
110
+ self.set_weights(global_weights)
111
+ logger.info("Updated local model with global weights")
112
+
113
+ # Generate/load local data
114
  X, y = self._generate_dummy_data()
115
+ logger.info(f"Training on {len(X)} samples")
116
 
117
  # Train locally
 
118
  history = self.train_local((X, y))
119
 
120
+ # Prepare metrics
121
+ metrics = {
122
+ 'dataset_size': len(X),
123
+ 'final_loss': history['loss'][-1] if history['loss'] else 0.0,
124
+ 'epochs_trained': len(history['loss']),
125
+ 'round': round_num
126
+ }
127
+
128
+ # Submit update to server
129
+ local_weights = self.get_weights()
130
+ self.http_client.submit_model_update(local_weights, metrics)
131
+
132
+ logger.info(f"Round {round_num} completed - Final loss: {metrics['final_loss']:.4f}")
 
 
 
 
 
 
 
133
 
134
  except Exception as e:
135
+ logger.error(f"Error in round {round_num}: {str(e)}")
136
  raise
137
 
138
  def _generate_dummy_data(self):
139
  """Generate dummy data for testing."""
140
+ try:
141
+ # Try to use the data handler for more realistic data
142
+ return self.data_handler.generate_synthetic_data(100)
143
+ except Exception:
144
+ # Fallback to simple dummy data
145
+ num_samples = 100
146
+ input_dim = 32 # Match with model's input dimension
147
+
148
+ # Generate input data
149
+ X = tf.random.normal((num_samples, input_dim))
150
+ # Generate target data (for this example, we'll predict the sum of inputs)
151
+ y = tf.reduce_sum(X, axis=1, keepdims=True)
152
+
153
+ return X.numpy(), y.numpy()
154
 
155
  def _build_model(self):
156
  """Build the initial model architecture."""
 
174
  logger = logging.getLogger(__name__)
175
  X, y = data
176
 
177
+ # Ensure data is in the right format
178
+ if isinstance(X, np.ndarray):
179
+ X = tf.convert_to_tensor(X, dtype=tf.float32)
180
+ if isinstance(y, np.ndarray):
181
+ y = tf.convert_to_tensor(y, dtype=tf.float32)
182
+
183
  # Log training parameters
184
+ logger.info(f"Training Parameters:")
 
185
  logger.info(f"Input shape: {X.shape}")
186
  logger.info(f"Output shape: {y.shape}")
187
+ logger.info(f"Batch size: {self.config.get('training', {}).get('batch_size', 32)}")
188
  logger.info(f"Epochs: {self.config.get('training', {}).get('local_epochs', 5)}")
 
 
189
 
190
  class LogCallback(tf.keras.callbacks.Callback):
191
  def on_epoch_end(self, epoch, logs=None):
192
+ logger.debug(f"Epoch {epoch + 1} - loss: {logs['loss']:.4f}")
193
 
194
+ # Train the model
195
  history = self.model.fit(
196
  X, y,
197
+ batch_size=self.config.get('training', {}).get('batch_size', 32),
198
+ epochs=self.config.get('training', {}).get('local_epochs', 3),
199
+ verbose=0,
200
+ callbacks=[LogCallback()]
201
  )
202
  return history.history
203
 
src/client/model_new.py ADDED
@@ -0,0 +1,214 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """model.py module."""
2
+
3
+ from typing import Dict, List, Optional, Tuple
4
+ import tensorflow as tf
5
+ import numpy as np
6
+ import logging
7
+ import time
8
+ from ..api.client import FederatedHTTPClient
9
+ from .data_handler import FinancialDataHandler
10
+
11
+ class FederatedClient:
12
+ def __init__(self, client_id: str, config: Dict, server_url: Optional[str] = None):
13
+ """Initialize the federated client."""
14
+ self.client_id = str(client_id)
15
+ self.config = config.get('client', {})
16
+ self.model = self._build_model()
17
+ self.data_handler = FinancialDataHandler(config)
18
+
19
+ # HTTP client for server communication
20
+ self.server_url = server_url or self.config.get('server_url', 'http://localhost:8080')
21
+ self.http_client = FederatedHTTPClient(self.server_url, self.client_id)
22
+
23
+ # Training state
24
+ self.registered = False
25
+ self.current_round = 0
26
+
27
+ def start(self):
28
+ """Start the federated client process with server communication."""
29
+ logger = logging.getLogger(__name__)
30
+ logger.info(f"Client {self.client_id} starting...")
31
+
32
+ try:
33
+ # Wait for server to be available
34
+ if not self.http_client.wait_for_server():
35
+ raise ConnectionError(f"Cannot connect to server at {self.server_url}")
36
+
37
+ # Register with server
38
+ self._register_with_server()
39
+
40
+ # Main federated learning loop
41
+ self._federated_learning_loop()
42
+
43
+ except Exception as e:
44
+ logger.error(f"Error during client execution: {str(e)}")
45
+ raise
46
+ finally:
47
+ self.http_client.close()
48
+
49
+ def _register_with_server(self):
50
+ """Register this client with the federated server"""
51
+ logger = logging.getLogger(__name__)
52
+
53
+ try:
54
+ # Generate local data to get client info
55
+ X, y = self._generate_dummy_data()
56
+
57
+ client_info = {
58
+ 'dataset_size': len(X),
59
+ 'model_params': self.model.count_params(),
60
+ 'capabilities': ['training', 'inference']
61
+ }
62
+
63
+ response = self.http_client.register(client_info)
64
+ self.registered = True
65
+
66
+ logger.info(f"Successfully registered with server")
67
+ logger.info(f"Dataset size: {client_info['dataset_size']}")
68
+ logger.info(f"Model parameters: {client_info['model_params']:,}")
69
+
70
+ except Exception as e:
71
+ logger.error(f"Failed to register with server: {str(e)}")
72
+ raise
73
+
74
+ def _federated_learning_loop(self):
75
+ """Main federated learning loop"""
76
+ logger = logging.getLogger(__name__)
77
+
78
+ while True:
79
+ try:
80
+ # Get training status from server
81
+ status = self.http_client.get_training_status()
82
+
83
+ if not status.get('training_active', True):
84
+ logger.info("Training completed on server")
85
+ break
86
+
87
+ server_round = status.get('current_round', 0)
88
+
89
+ if server_round > self.current_round:
90
+ self._participate_in_round(server_round)
91
+ self.current_round = server_round
92
+
93
+ time.sleep(5) # Check every 5 seconds
94
+
95
+ except Exception as e:
96
+ logger.error(f"Error in federated learning loop: {str(e)}")
97
+ time.sleep(10) # Wait longer on error
98
+
99
+ def _participate_in_round(self, round_num: int):
100
+ """Participate in a federated learning round"""
101
+ logger = logging.getLogger(__name__)
102
+ logger.info(f"Participating in round {round_num}")
103
+
104
+ try:
105
+ # Get global model from server
106
+ model_response = self.http_client.get_global_model()
107
+ global_weights = model_response.get('model_weights')
108
+
109
+ if global_weights:
110
+ self.set_weights(global_weights)
111
+ logger.info("Updated local model with global weights")
112
+
113
+ # Generate/load local data
114
+ X, y = self._generate_dummy_data()
115
+ logger.info(f"Training on {len(X)} samples")
116
+
117
+ # Train locally
118
+ history = self.train_local((X, y))
119
+
120
+ # Prepare metrics
121
+ metrics = {
122
+ 'dataset_size': len(X),
123
+ 'final_loss': history['loss'][-1] if history['loss'] else 0.0,
124
+ 'epochs_trained': len(history['loss']),
125
+ 'round': round_num
126
+ }
127
+
128
+ # Submit update to server
129
+ local_weights = self.get_weights()
130
+ self.http_client.submit_model_update(local_weights, metrics)
131
+
132
+ logger.info(f"Round {round_num} completed - Final loss: {metrics['final_loss']:.4f}")
133
+
134
+ except Exception as e:
135
+ logger.error(f"Error in round {round_num}: {str(e)}")
136
+ raise
137
+
138
+ def _generate_dummy_data(self):
139
+ """Generate dummy data for testing."""
140
+ try:
141
+ # Try to use the data handler for more realistic data
142
+ return self.data_handler.generate_synthetic_data(100)
143
+ except Exception:
144
+ # Fallback to simple dummy data
145
+ num_samples = 100
146
+ input_dim = 32 # Match with model's input dimension
147
+
148
+ # Generate input data
149
+ X = tf.random.normal((num_samples, input_dim))
150
+ # Generate target data (for this example, we'll predict the sum of inputs)
151
+ y = tf.reduce_sum(X, axis=1, keepdims=True)
152
+
153
+ return X.numpy(), y.numpy()
154
+
155
+ def _build_model(self):
156
+ """Build the initial model architecture."""
157
+ input_dim = 32 # Match with data generation
158
+ model = tf.keras.Sequential([
159
+ tf.keras.layers.Input(shape=(input_dim,)),
160
+ tf.keras.layers.Dense(128, activation='relu'),
161
+ tf.keras.layers.Dense(64, activation='relu'),
162
+ tf.keras.layers.Dense(1) # Output layer for regression
163
+ ])
164
+ model.compile(
165
+ optimizer=tf.keras.optimizers.Adam(
166
+ learning_rate=self.config.get('training', {}).get('learning_rate', 0.001)
167
+ ),
168
+ loss='mse'
169
+ )
170
+ return model
171
+
172
+ def train_local(self, data):
173
+ """Train the model on local data."""
174
+ logger = logging.getLogger(__name__)
175
+ X, y = data
176
+
177
+ # Ensure data is in the right format
178
+ if isinstance(X, np.ndarray):
179
+ X = tf.convert_to_tensor(X, dtype=tf.float32)
180
+ if isinstance(y, np.ndarray):
181
+ y = tf.convert_to_tensor(y, dtype=tf.float32)
182
+
183
+ # Log training parameters
184
+ logger.info(f"Training Parameters:")
185
+ logger.info(f"Input shape: {X.shape}")
186
+ logger.info(f"Output shape: {y.shape}")
187
+ logger.info(f"Batch size: {self.config.get('training', {}).get('batch_size', 32)}")
188
+ logger.info(f"Epochs: {self.config.get('training', {}).get('local_epochs', 5)}")
189
+
190
+ class LogCallback(tf.keras.callbacks.Callback):
191
+ def on_epoch_end(self, epoch, logs=None):
192
+ logger.debug(f"Epoch {epoch + 1} - loss: {logs['loss']:.4f}")
193
+
194
+ # Train the model
195
+ history = self.model.fit(
196
+ X, y,
197
+ batch_size=self.config.get('training', {}).get('batch_size', 32),
198
+ epochs=self.config.get('training', {}).get('local_epochs', 3),
199
+ verbose=0,
200
+ callbacks=[LogCallback()]
201
+ )
202
+ return history.history
203
+
204
+ def get_weights(self) -> List:
205
+ """Get the model weights."""
206
+ weights = self.model.get_weights()
207
+ # Convert to serializable format
208
+ return [w.tolist() for w in weights]
209
+
210
+ def set_weights(self, weights: List):
211
+ """Update local model with global weights."""
212
+ # Convert from serializable format back to numpy arrays
213
+ np_weights = [np.array(w) for w in weights]
214
+ self.model.set_weights(np_weights)
src/main.py CHANGED
@@ -51,11 +51,15 @@ def main():
51
 
52
  if args.mode == 'server':
53
  coordinator = FederatedCoordinator(config)
54
- logger.info("Starting server...")
55
  coordinator.start()
56
  else:
57
- client = FederatedClient(1, config)
58
- logger.info(f"Starting client with ID: {client.client_id}")
 
 
 
 
59
  client.start()
60
 
61
  if __name__ == "__main__":
 
51
 
52
  if args.mode == 'server':
53
  coordinator = FederatedCoordinator(config)
54
+ logger.info("Starting federated server...")
55
  coordinator.start()
56
  else:
57
+ # Extract client ID from config or use default
58
+ client_id = config.get('client', {}).get('id', '1')
59
+ server_url = config.get('client', {}).get('server_url', 'http://localhost:8080')
60
+
61
+ client = FederatedClient(client_id, config, server_url)
62
+ logger.info(f"Starting federated client with ID: {client_id}")
63
  client.start()
64
 
65
  if __name__ == "__main__":
src/server/aggregator.py CHANGED
@@ -4,40 +4,56 @@ import tensorflow as tf
4
  from typing import List, Dict
5
  import numpy as np
6
  from collections import defaultdict
 
7
 
8
  class FederatedAggregator:
9
  def __init__(self, config: Dict):
10
- """Initialize the federated aggregator."""
11
- self.weighted = config['aggregation']['weighted']
12
-
 
 
 
 
 
 
 
 
 
 
 
13
  def compute_metrics(self, client_metrics: List[Dict]) -> Dict:
14
- """Compute aggregated metrics from client updates."""
 
15
  if not client_metrics:
 
16
  return {}
17
-
18
  aggregated_metrics = defaultdict(float)
19
  total_samples = sum(metrics['num_samples'] for metrics in client_metrics)
20
-
21
  for metrics in client_metrics:
22
  weight = metrics['num_samples'] / total_samples if self.weighted else 1.0
23
-
24
  for metric_name, value in metrics['metrics'].items():
25
  aggregated_metrics[metric_name] += value * weight
26
-
27
  return dict(aggregated_metrics)
28
 
29
  def check_convergence(self,
30
  old_weights: List,
31
  new_weights: List,
32
  threshold: float = 1e-5) -> bool:
33
- """Check if the model has converged."""
 
34
  if old_weights is None or new_weights is None:
 
35
  return False
36
-
37
  weight_differences = [
38
  np.mean(np.abs(old - new))
39
  for old, new in zip(old_weights, new_weights)
40
  ]
41
-
42
- return all(diff < threshold for diff in weight_differences)
 
 
43
 
 
4
  from typing import List, Dict
5
  import numpy as np
6
  from collections import defaultdict
7
+ import logging
8
 
9
  class FederatedAggregator:
10
  def __init__(self, config: Dict):
11
+ logger = logging.getLogger(__name__)
12
+ logger.debug(f"Initializing FederatedAggregator with config: {config}")
13
+ # Defensive: try to find aggregation config
14
+ agg_config = None
15
+ if 'aggregation' in config:
16
+ agg_config = config['aggregation']
17
+ elif 'server' in config and 'aggregation' in config['server']:
18
+ agg_config = config['server']['aggregation']
19
+ else:
20
+ logger.error(f"No 'aggregation' key found in config passed to FederatedAggregator: {config}")
21
+ raise KeyError("'aggregation' config section is required for FederatedAggregator")
22
+ self.weighted = agg_config.get('weighted', True)
23
+ logger.info(f"FederatedAggregator initialized. Weighted: {self.weighted}")
24
+
25
  def compute_metrics(self, client_metrics: List[Dict]) -> Dict:
26
+ logger = logging.getLogger(__name__)
27
+ logger.debug(f"Computing metrics for {len(client_metrics)} clients")
28
  if not client_metrics:
29
+ logger.warning("No client metrics provided to compute_metrics.")
30
  return {}
 
31
  aggregated_metrics = defaultdict(float)
32
  total_samples = sum(metrics['num_samples'] for metrics in client_metrics)
33
+ logger.debug(f"Total samples across clients: {total_samples}")
34
  for metrics in client_metrics:
35
  weight = metrics['num_samples'] / total_samples if self.weighted else 1.0
36
+ logger.debug(f"Client metrics: {metrics}, weight: {weight}")
37
  for metric_name, value in metrics['metrics'].items():
38
  aggregated_metrics[metric_name] += value * weight
39
+ logger.info(f"Aggregated metrics: {dict(aggregated_metrics)}")
40
  return dict(aggregated_metrics)
41
 
42
  def check_convergence(self,
43
  old_weights: List,
44
  new_weights: List,
45
  threshold: float = 1e-5) -> bool:
46
+ logger = logging.getLogger(__name__)
47
+ logger.debug("Checking convergence...")
48
  if old_weights is None or new_weights is None:
49
+ logger.warning("Old or new weights are None in check_convergence.")
50
  return False
 
51
  weight_differences = [
52
  np.mean(np.abs(old - new))
53
  for old, new in zip(old_weights, new_weights)
54
  ]
55
+ logger.debug(f"Weight differences: {weight_differences}")
56
+ converged = all(diff < threshold for diff in weight_differences)
57
+ logger.info(f"Convergence status: {converged}")
58
+ return converged
59
 
src/server/coordinator.py CHANGED
@@ -1,49 +1,129 @@
1
  """coordinator.py module."""
2
 
3
  import tensorflow as tf
4
- from typing import List, Dict
5
  import numpy as np
6
  from collections import defaultdict
7
  import logging
8
  import time
 
 
9
 
10
  class FederatedCoordinator:
11
  def __init__(self, config: Dict):
12
  """Initialize the federated learning coordinator."""
 
 
13
  self.config = config
14
  self.clients = {}
 
 
15
  self.current_round = 0
 
16
  self.min_clients = config.get('server', {}).get('federated', {}).get('min_clients', 2)
17
  self.rounds = config.get('server', {}).get('federated', {}).get('rounds', 10)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
 
19
- def register_client(self, client_id: int, client_size: int):
20
  """Register a new client."""
21
- self.clients[client_id] = {
22
- 'size': client_size,
23
- 'weights': None,
24
- 'metrics': defaultdict(list)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
  }
26
 
27
- def aggregate_weights(self, client_updates: List[Dict]) -> List:
28
- """Aggregate weights using FedAvg algorithm."""
29
- total_size = sum(self.clients[update['client_id']]['size']
30
- for update in client_updates)
31
-
32
- aggregated_weights = [
33
- np.zeros_like(w) for w in client_updates[0]['weights']
34
- ]
35
-
36
- for update in client_updates:
37
- client_size = self.clients[update['client_id']]['size']
38
- weight = client_size / total_size
39
 
40
- for i, layer_weights in enumerate(update['weights']):
41
- aggregated_weights[i] += layer_weights * weight
42
-
43
- return aggregated_weights
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
44
 
45
  def start(self):
46
- """Start the federated learning process."""
47
  logger = logging.getLogger(__name__)
48
 
49
  # Print server startup information
@@ -56,22 +136,44 @@ class FederatedCoordinator:
56
  logger.info("-" * 30)
57
  logger.info(f"Minimum clients required: {self.min_clients}")
58
  logger.info(f"Total rounds planned: {self.rounds}")
59
- logger.info(f"Current active clients: {len(self.clients)}")
 
60
  logger.info("-" * 30 + "\n")
61
 
62
- while self.current_round < self.rounds:
63
- round_num = self.current_round + 1
64
- logger.info(f"\nRound {round_num}/{self.rounds}")
65
- logger.info("-" * 30)
 
66
 
67
- if len(self.clients) < self.min_clients:
68
- logger.warning(
69
- f"Waiting for clients... "
70
- f"(active: {len(self.clients)}/{self.min_clients})"
71
- )
72
- time.sleep(5)
73
- continue
74
 
75
- logger.info(f"Active clients: {list(self.clients.keys())}")
76
- logger.info(f"Starting training round {round_num}")
77
- self.current_round += 1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  """coordinator.py module."""
2
 
3
  import tensorflow as tf
4
+ from typing import List, Dict, Any, Optional
5
  import numpy as np
6
  from collections import defaultdict
7
  import logging
8
  import time
9
+ import threading
10
+ from .aggregator import FederatedAggregator
11
 
12
  class FederatedCoordinator:
13
  def __init__(self, config: Dict):
14
  """Initialize the federated learning coordinator."""
15
+ logger = logging.getLogger(__name__)
16
+ logger.debug(f"Initializing FederatedCoordinator with config: {config}")
17
  self.config = config
18
  self.clients = {}
19
+ self.client_updates = {} # Store updates for current round
20
+ self.global_model_weights = None
21
  self.current_round = 0
22
+ self.training_active = False
23
  self.min_clients = config.get('server', {}).get('federated', {}).get('min_clients', 2)
24
  self.rounds = config.get('server', {}).get('federated', {}).get('rounds', 10)
25
+ # Debug: log config structure
26
+ logger.debug(f"Coordinator received config: {config}")
27
+ # Robustly extract aggregation config
28
+ agg_config = None
29
+ if 'aggregation' in config:
30
+ agg_config = config
31
+ elif 'server' in config and 'aggregation' in config['server']:
32
+ agg_config = config['server']
33
+ else:
34
+ logger.error(f"No 'aggregation' key found in config for FederatedAggregator: {config}")
35
+ raise ValueError("'aggregation' config section is required for FederatedAggregator")
36
+ logger.debug(f"Passing aggregation config to FederatedAggregator: {agg_config}")
37
+ try:
38
+ self.aggregator = FederatedAggregator(agg_config)
39
+ except Exception as e:
40
+ logger.error(f"Error initializing FederatedAggregator: {e}")
41
+ raise
42
+ self.lock = threading.Lock() # Thread safety for concurrent API calls
43
+ logger.info("FederatedCoordinator initialized.")
44
 
45
+ def register_client(self, client_id: str, client_info: Dict[str, Any] = None) -> bool:
46
  """Register a new client."""
47
+ with self.lock:
48
+ if client_id in self.clients:
49
+ logging.getLogger(__name__).warning(f"Client {client_id} already registered")
50
+ return True
51
+
52
+ self.clients[client_id] = {
53
+ 'info': client_info or {},
54
+ 'last_seen': time.time(),
55
+ 'metrics': defaultdict(list)
56
+ }
57
+
58
+ logging.getLogger(__name__).info(f"Client {client_id} registered successfully")
59
+ return True
60
+
61
+ def get_client_config(self) -> Dict[str, Any]:
62
+ """Get configuration to send to clients"""
63
+ return {
64
+ 'model_config': self.config.get('model', {}),
65
+ 'training_config': self.config.get('training', {}),
66
+ 'current_round': self.current_round,
67
+ 'total_rounds': self.rounds
68
  }
69
 
70
+ def get_global_model(self) -> Optional[List]:
71
+ """Get the current global model weights"""
72
+ with self.lock:
73
+ return self.global_model_weights
74
+
75
+ def receive_model_update(self, client_id: str, model_weights: List, metrics: Dict[str, Any]):
76
+ """Receive a model update from a client"""
77
+ with self.lock:
78
+ if client_id not in self.clients:
79
+ raise ValueError(f"Client {client_id} not registered")
 
 
80
 
81
+ self.client_updates[client_id] = {
82
+ 'weights': model_weights,
83
+ 'metrics': metrics,
84
+ 'timestamp': time.time()
85
+ }
86
+
87
+ self.clients[client_id]['last_seen'] = time.time()
88
+
89
+ logger = logging.getLogger(__name__)
90
+ logger.info(f"Received update from client {client_id}")
91
+
92
+ # Check if we have enough updates for aggregation
93
+ if len(self.client_updates) >= self.min_clients:
94
+ self._aggregate_models()
95
+
96
+ def _aggregate_models(self):
97
+ """Aggregate models from all client updates"""
98
+ try:
99
+ logger = logging.getLogger(__name__)
100
+ logger.info(f"Aggregating models from {len(self.client_updates)} clients")
101
+
102
+ # Prepare updates for aggregation
103
+ updates = []
104
+ for client_id, update in self.client_updates.items():
105
+ client_size = update['metrics'].get('dataset_size', 100) # Default size
106
+ updates.append({
107
+ 'client_id': client_id,
108
+ 'weights': update['weights'],
109
+ 'size': client_size
110
+ })
111
+
112
+ # Aggregate using FedAvg
113
+ self.global_model_weights = self.aggregator.federated_averaging(updates)
114
+
115
+ # Clear updates for next round
116
+ self.client_updates.clear()
117
+ self.current_round += 1
118
+
119
+ logger.info(f"Model aggregation completed for round {self.current_round}")
120
+
121
+ except Exception as e:
122
+ logger = logging.getLogger(__name__)
123
+ logger.error(f"Error during model aggregation: {str(e)}")
124
 
125
  def start(self):
126
+ """Start the federated learning process with API server"""
127
  logger = logging.getLogger(__name__)
128
 
129
  # Print server startup information
 
136
  logger.info("-" * 30)
137
  logger.info(f"Minimum clients required: {self.min_clients}")
138
  logger.info(f"Total rounds planned: {self.rounds}")
139
+ active_clients_count = self._count_active_clients()
140
+ logger.info(f"Current active clients: {active_clients_count}")
141
  logger.info("-" * 30 + "\n")
142
 
143
+ self.training_active = True
144
+
145
+ # Import and start API server
146
+ try:
147
+ from ..api.server import FederatedAPI
148
 
149
+ api_config = self.config.get('server', {}).get('api', {})
150
+ host = api_config.get('host', '0.0.0.0')
151
+ port = api_config.get('port', 8080)
 
 
 
 
152
 
153
+ api_server = FederatedAPI(self, host, port)
154
+ api_thread = api_server.run_threaded()
155
+
156
+ logger.info(f"API server started on {host}:{port}")
157
+
158
+ # Keep server running
159
+ try:
160
+ while self.training_active and self.current_round < self.rounds:
161
+ time.sleep(1) # Keep main thread alive
162
+
163
+ # Log progress periodically
164
+ active_clients_count = self._count_active_clients()
165
+ if active_clients_count > 0:
166
+ logger.debug(f"Round {self.current_round}/{self.rounds}, "
167
+ f"Active Clients: {active_clients_count}, "
168
+ f"Updates: {len(self.client_updates)}")
169
+
170
+ logger.info("Federated learning completed successfully")
171
+
172
+ except KeyboardInterrupt:
173
+ logger.info("Server shutdown requested")
174
+ self.training_active = False
175
+
176
+ except ImportError as e:
177
+ logger.error(f"Failed to start API server: {str(e)}")
178
+ # Fallback to original behavior
179
+ # ...existing code...
src/utils/metrics.py CHANGED
@@ -6,6 +6,13 @@ from scipy.stats import wasserstein_distance, ks_2samp
6
  from sklearn.metrics import mutual_info_score, silhouette_score
7
  from sklearn.neighbors import NearestNeighbors
8
 
 
 
 
 
 
 
 
9
  class MetricsCalculator:
10
  @staticmethod
11
  def calculate_distribution_similarity(real_data: np.ndarray,
 
6
  from sklearn.metrics import mutual_info_score, silhouette_score
7
  from sklearn.neighbors import NearestNeighbors
8
 
9
+ def calculate_model_similarity(model_a, model_b):
10
+ """
11
+ Placeholder for model similarity calculation.
12
+ Returns a dummy similarity score.
13
+ """
14
+ return 1.0 # Always returns perfect similarity for now
15
+
16
  class MetricsCalculator:
17
  @staticmethod
18
  def calculate_distribution_similarity(real_data: np.ndarray,
test_implementation.py ADDED
@@ -0,0 +1,157 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Simple test script for the federated learning implementation
4
+ """
5
+
6
+ import sys
7
+ import time
8
+ import subprocess
9
+ import threading
10
+ import os
11
+ from pathlib import Path
12
+ import logging
13
+ import yaml
14
+
15
+ # Set up debug logging for the test
16
+ logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s')
17
+
18
+ # Add src to path
19
+ sys.path.append(str(Path(__file__).parent / "src"))
20
+
21
+ def load_client_config():
22
+ config_path = Path(__file__).parent / "config" / "client_config.yaml"
23
+ with open(config_path, 'r') as f:
24
+ full_config = yaml.safe_load(f)
25
+ return full_config
26
+
27
+ def test_basic_functionality():
28
+ """Test basic federated learning functionality"""
29
+ print("Testing FinFedRAG Basic Functionality")
30
+ print("=" * 50)
31
+
32
+ # Test 1: Import all modules
33
+ print("Test 1: Testing imports...")
34
+ try:
35
+ from src.server.coordinator import FederatedCoordinator
36
+ from src.client.model import FederatedClient
37
+ from src.api.server import FederatedAPI
38
+ from src.api.client import FederatedHTTPClient
39
+ print("✓ All imports successful")
40
+ logging.debug("All modules imported successfully.")
41
+ except ImportError as e:
42
+ print(f"✗ Import failed: {e}")
43
+ logging.error(f"Import failed: {e}")
44
+ return False
45
+
46
+ # Test 2: Create coordinator
47
+ print("\nTest 2: Testing coordinator creation...")
48
+ try:
49
+ config = {
50
+ 'server': {
51
+ 'federated': {'min_clients': 2, 'rounds': 3},
52
+ 'api': {'host': 'localhost', 'port': 8081},
53
+ 'aggregation': {'method': 'fedavg', 'weighted': True}
54
+ },
55
+ 'model': {'input_dim': 32},
56
+ 'training': {'learning_rate': 0.001}
57
+ }
58
+ logging.debug(f"Coordinator test config: {config}")
59
+ coordinator = FederatedCoordinator(config)
60
+ print("✓ Coordinator created successfully")
61
+ logging.debug("Coordinator created successfully.")
62
+ except Exception as e:
63
+ print(f"✗ Coordinator creation failed: {e}")
64
+ logging.error(f"Coordinator creation failed: {e}")
65
+ return False
66
+
67
+ # Test 3: Create client
68
+ print("\nTest 3: Testing client creation...")
69
+ try:
70
+ client_config = load_client_config()
71
+ logging.debug(f"Client test config: {client_config}")
72
+ client = FederatedClient("test_client", client_config)
73
+ print("✓ Client created successfully")
74
+ logging.debug("Client created successfully.")
75
+ except Exception as e:
76
+ print(f"✗ Client creation failed: {e}")
77
+ logging.error(f"Client creation failed: {e}")
78
+ return False
79
+
80
+ # Test 4: Test HTTP client
81
+ print("\nTest 4: Testing HTTP client...")
82
+ try:
83
+ http_client = FederatedHTTPClient('http://localhost:8081', 'test_client')
84
+ print("✓ HTTP client created successfully")
85
+ logging.debug("HTTP client created successfully.")
86
+ except Exception as e:
87
+ print(f"✗ HTTP client creation failed: {e}")
88
+ logging.error(f"HTTP client creation failed: {e}")
89
+ return False
90
+
91
+ print("\n" + "=" * 50)
92
+ print("All basic functionality tests passed!")
93
+ logging.debug("All basic functionality tests passed.")
94
+ return True
95
+
96
+ def run_integration_test():
97
+ """Run a quick integration test"""
98
+ print("\nRunning Integration Test")
99
+ print("=" * 50)
100
+
101
+ # This would start a server and client in separate processes
102
+ # For now, just validate the configuration files
103
+
104
+ config_dir = Path("config")
105
+
106
+ # Test server config
107
+ server_config = config_dir / "server_config.yaml"
108
+ if server_config.exists():
109
+ print("✓ Server config exists")
110
+ logging.debug("Server config exists.")
111
+ else:
112
+ print("✗ Server config missing")
113
+ logging.error("Server config missing.")
114
+ return False
115
+
116
+ # Test client config
117
+ client_config = config_dir / "client_config.yaml"
118
+ if client_config.exists():
119
+ print("✓ Client config exists")
120
+ logging.debug("Client config exists.")
121
+ else:
122
+ print("✗ Client config missing")
123
+ logging.error("Client config missing.")
124
+ return False
125
+
126
+ print("✓ Configuration files are present")
127
+ print("✓ Integration test setup complete")
128
+ logging.debug("Integration test setup complete.")
129
+
130
+ return True
131
+
132
+ if __name__ == "__main__":
133
+ print("FinFedRAG Test Suite")
134
+ print("=" * 50)
135
+
136
+ # Change to project directory
137
+ os.chdir(Path(__file__).parent)
138
+
139
+ success = True
140
+
141
+ # Run basic functionality tests
142
+ if not test_basic_functionality():
143
+ success = False
144
+
145
+ # Run integration tests
146
+ if not run_integration_test():
147
+ success = False
148
+
149
+ print("\n" + "=" * 50)
150
+ if success:
151
+ print("🎉 All tests passed!")
152
+ print("\nTo run the system:")
153
+ print("1. Start server: python -m src.main --mode server --config config/server_config.yaml")
154
+ print("2. Start client: python -m src.main --mode client --config config/client_config.yaml")
155
+ else:
156
+ print("❌ Some tests failed!")
157
+ sys.exit(1)