fix

downloads-generation/models_class1_pan_unselected/generate_hyperparameters.py

@@ -16,7 +16,7 @@ base_hyperparameters = {
     'early_stopping': True,
     'init': 'glorot_uniform',
     'layer_sizes': [1024, 512],
-    'learning_rate': None,
+    'learning_rate': 0.001,
     'locally_connected_layers': [],
     'loss': 'custom:mse_with_inequalities',
     'max_epochs': 5000,
@@ -59,11 +59,13 @@ for layer_sizes in [[512, 256], [512, 512], [1024, 512], [1024, 1024]]:
     for pretrain in [True]:
         l1_base = 0.0000001
         for l1 in [l1_base, l1_base / 10, l1_base / 100, l1_base / 1000, 0.0]:
-            new = deepcopy(base_hyperparameters)
-            new["layer_sizes"] = layer_sizes
-            new["dense_layer_l1_regularization"] = l1
-            new["train_data"]["pretrain"] = pretrain
-            if not grid or new not in grid:
-                grid.append(new)
+            for lr in [0.001, 0.01]:
+                new = deepcopy(base_hyperparameters)
+                new["layer_sizes"] = layer_sizes
+                new["dense_layer_l1_regularization"] = l1
+                new["train_data"]["pretrain"] = pretrain
+                new["learning_rate"] = lr
+                if not grid or new not in grid:
+                    grid.append(new)
 
 dump(grid, stdout)

mhcflurry/class1_neural_network.py

@@ -602,9 +602,8 @@ class Class1NeuralNetwork(object):
                 epochs, max(min_val_loss_iteration + patience, min_epochs))
 
             progress_message = (
-                "epoch %3d / %3d [%0.2f sec.]: loss=%g val_loss=%g. Min val "
-                "loss (%g) at epoch %s. Cumulative training points: %d. "
-                "Earliest stop epoch: %d." % (
+                "epoch %3d/%3d [%0.2f sec.]: loss=%g val_loss=%g. Min val "
+                "loss %g at epoch %s. Cum. points: %d. Stop at epoch %d." % (
                     epoch,
                     epochs,
                     epoch_time,

mhcflurry/cluster_parallelism.py

@@ -139,16 +139,18 @@ def cluster_results(
     def result_generator():
         start = time.time()
         while result_items:
+            print("[%0.1f sec elapsed] waiting on %d / %d items." % (
+                time.time() - start, len(result_items), len(work_items)))
             while True:
                 result_item = None
                 for d in result_items:
-                    if os.path.exists(item['finished_path']):
+                    if os.path.exists(d['finished_path']):
                         result_item = d
                         break
                 if result_item is None:
                     os.sleep(60)
                 else:
-                    del result_items[result_item]
+                    result_items.remove(result_item)
                     break
 
             complete_dir = result_item['finished_path']

test/test_train_pan_allele_models_command.py

@@ -163,4 +163,5 @@ def test_run_cluster_parallelism():
 
 
 if __name__ == "__main__":
-    run_and_check(n_jobs=0, delete=False)
+    #run_and_check(n_jobs=0, delete=False)
+    test_run_cluster_parallelism()