Saving weights and logs of step 500

events.out.tfevents.1625836684.t1v-n-112df4a9-w-0.38631.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1198c8afc6504eecba539da0dee931919669a28655dbd66157e980a6606dbd50
+size 11274

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76d55c57630984430801c3b6d5cc75703bd2a61ef6bb8f9d20f7c6150f2dfbd3
+oid sha256:f1f30bd95331d281308f6cb94e74bec0d82ebc06684501d536c8d38952e09abb
 size 498796983

run.log

@@ -1,4 +1,5 @@
-[10:39:50] - INFO - __main__ - Training/evaluation parameters TrainingArguments(
+2021-07-09 13:17:54.997560: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory
+[13:17:56] - INFO - __main__ - Training/evaluation parameters TrainingArguments(
 _n_gpu=-1,
 adafactor=False,
 adam_beta1=0.9,
@@ -34,7 +35,7 @@ local_rank=-1,
 log_level=-1,
 log_level_replica=-1,
 log_on_each_node=True,
-logging_dir=./runs/Jul09_10-39-50_t1v-n-112df4a9-w-0,
+logging_dir=./runs/Jul09_13-17-56_t1v-n-112df4a9-w-0,
 logging_first_step=False,
 logging_steps=500,
 logging_strategy=IntervalStrategy.STEPS,
@@ -73,326 +74,181 @@ warmup_ratio=0.0,
 warmup_steps=1000,
 weight_decay=0.0,
 )
-[10:39:50] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
-[10:39:50] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
-[10:39:50] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
-[10:39:50] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
-[10:39:50] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
-[10:39:50] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
-[10:39:50] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
-[10:39:50] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
-[10:39:51] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
-[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
-[10:39:51] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
-[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
-[10:39:51] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
-[10:39:51] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
-[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
-[10:39:51] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
-[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
-[10:39:51] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
-[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
-[10:39:51] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
-[10:39:51] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
-[10:39:51] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-0f52086e7b10d7e8.arrow
-[10:39:51] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-a39e5f5a5c6c69fc.arrow
-[10:39:51] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-e4d3282a2dd50fa0.arrow
-[10:39:51] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-b9a3aa9913be3b34.arrow
-[10:39:51] - INFO - absl - Starting the local TPU driver.
-[10:39:51] - INFO - absl - Unable to initialize backend 'tpu_driver': Not found: Unable to find driver in registry given worker: local://
-[10:39:51] - INFO - absl - Unable to initialize backend 'gpu': Not found: Could not find registered platform with name: "cuda". Available platform names are: Interpreter Host TPU
-[10:39:55] - WARNING - __main__ - Unable to display metrics through TensorBoard because some package are not installed: No module named 'tensorflow'
+[13:17:56] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
+[13:17:56] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
+[13:17:56] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
+[13:17:56] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
+[13:17:56] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
+[13:17:56] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
+[13:17:56] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
+[13:17:56] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
+[13:17:57] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
+[13:17:57] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
+[13:17:57] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
+[13:17:57] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
+[13:17:57] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
+[13:17:57] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
+[13:17:57] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
+[13:17:57] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
+[13:17:57] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
+[13:17:57] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
+[13:17:57] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
+[13:17:57] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
+[13:17:57] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
+[13:17:57] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-0f52086e7b10d7e8.arrow
+[13:17:57] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-a39e5f5a5c6c69fc.arrow
+
  0%|          | 0/5 [00:00<?, ?ba/s]
 20%|██        | 1/5 [00:00<00:03,  1.33ba/s]
 40%|████      | 2/5 [00:01<00:02,  1.25ba/s]
 60%|██████    | 3/5 [00:02<00:01,  1.22ba/s]
 80%|████████  | 4/5 [00:03<00:00,  1.14ba/s]
+
  0%|          | 0/1 [00:00<?, ?ba/s]
+[13:18:01] - INFO - absl - Starting the local TPU driver.
+[13:18:01] - INFO - absl - Unable to initialize backend 'tpu_driver': Not found: Unable to find driver in registry given worker: local://
+[13:18:01] - INFO - absl - Unable to initialize backend 'gpu': Not found: Could not find registered platform with name: "cuda". Available platform names are: Interpreter Host TPU
+2021-07-09 13:18:04.352179: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory
+2021-07-09 13:18:04.352213: W tensorflow/stream_executor/cuda/cuda_driver.cc:326] failed call to cuInit: UNKNOWN ERROR (303)
 /home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
   warnings.warn(
 /home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
   warnings.warn(
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
                                                              
-
                                                             
-
-
-
-
-
-
-
-
-[10:41:35] - INFO - huggingface_hub.repository - git version 2.25.1
-git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
-[10:41:35] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
-
-
-
-
-
+
+
+
+
+
 
-
                                                                                                               
-
                                                            
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                              
+
                                                             
+
 
 
 
 
-
-[10:42:04] - INFO - huggingface_hub.repository - git version 2.25.1
+
+
+
+[13:19:46] - INFO - huggingface_hub.repository - git version 2.25.1
 git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
-[10:42:04] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
+[13:19:47] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo

run.sh

@@ -17,4 +17,4 @@ python run_mlm_flax.py \
     --run_name hf-flax-robert-base-mr \
     --push_to_hub_model_id="flax-community/roberta-base-mr" \
     --push_to_hub_token=$HUB_TOKEN \
-    --push_to_hub 2>&1 | tee run.log
+    --push_to_hub 2>&1 | tee run.log

run_mlm_flax.py

@@ -431,7 +431,8 @@ if __name__ == "__main__":
             total_length = len(concatenated_examples[list(examples.keys())[0]])
             # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
             # customize this part to your needs.
-            total_length = (total_length // max_seq_length) * max_seq_length
+            if total_length >= max_seq_length:
+                total_length = (total_length // max_seq_length) * max_seq_length
             # Split by chunks of max_len.
             result = {
                 k: [t[i : i + max_seq_length] for i in range(0, total_length, max_seq_length)]
@@ -457,6 +458,7 @@ if __name__ == "__main__":
     if has_tensorboard and jax.process_index() == 0:
         try:
             from flax.metrics.tensorboard import SummaryWriter
+
             summary_writer = SummaryWriter(log_dir=Path(training_args.output_dir))
         except ImportError as ie:
             has_tensorboard = False
@@ -477,7 +479,14 @@ if __name__ == "__main__":
     rng = jax.random.PRNGKey(training_args.seed)
     dropout_rngs = jax.random.split(rng, jax.local_device_count())
 
-    model = FlaxAutoModelForMaskedLM.from_config(config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype))
+    if model_args.model_name_or_path:
+        model = FlaxAutoModelForMaskedLM.from_pretrained(
+            model_args.model_name_or_path, config=config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype)
+        )
+    else:
+        model = FlaxAutoModelForMaskedLM.from_config(
+            config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype)
+        )
 
     # Store some constant
     num_epochs = int(training_args.num_train_epochs)
@@ -654,7 +663,6 @@ if __name__ == "__main__":
 
                 # Save metrics
                 if has_tensorboard and jax.process_index() == 0:
-                    cur_step = epoch * (len(tokenized_datasets["train"]) // train_batch_size)
                     write_eval_metric(summary_writer, eval_metrics, cur_step)
 
             if cur_step % training_args.save_steps == 0 and cur_step > 0:

test_mr_tokenizer.py

@@ -0,0 +1,8 @@
+from tokenizers import Tokenizer
+
+tokenizer = Tokenizer.from_file("tokenizer.json")
+
+text = "गेल्या अनेक दिवसांपासून केंद्र सरकरा विरुद्ध ट्विटर इंडिया यांच्यामध्ये सरकारच्या नव्या मार्गदर्शक सूचना किंवा नियमावलीवरून वाद सुरू आहे."
+
+output = tokenizer.encode(text)
+print(tokenizer.decode(output.ids))

wandb/debug-internal.log

@@ -1 +1 @@
-run-20210709_103433-d7n9baaw/logs/debug-internal.log
+run-20210709_125049-89spqu1d/logs/debug-internal.log

wandb/debug.log

@@ -1 +1 @@
-run-20210709_103433-d7n9baaw/logs/debug.log
+run-20210709_125049-89spqu1d/logs/debug.log

wandb/latest-run

@@ -1 +1 @@
-run-20210709_103433-d7n9baaw
+run-20210709_125049-89spqu1d

wandb/run-20210709_111555-275xd40l/files/config.yaml

@@ -0,0 +1,305 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.10.33
+    framework: huggingface
+    huggingface_version: 4.9.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.8.10
+    t:
+      1:
+      - 11
+      2:
+      - 11
+      4: 3.8.10
+      5: 0.10.33
+      6: 4.9.0.dev0
+      8:
+      - 5
+adafactor:
+  desc: null
+  value: false
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+cache_dir:
+  desc: null
+  value: null
+config_name:
+  desc: null
+  value: ./
+dataloader_drop_last:
+  desc: null
+  value: false
+dataloader_num_workers:
+  desc: null
+  value: 0
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataset_config_name:
+  desc: null
+  value: unshuffled_deduplicated_als
+dataset_name:
+  desc: null
+  value: oscar
+ddp_find_unused_parameters:
+  desc: null
+  value: null
+debug:
+  desc: null
+  value: []
+deepspeed:
+  desc: null
+  value: null
+disable_tqdm:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: false
+do_predict:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+dtype:
+  desc: null
+  value: float32
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_steps:
+  desc: null
+  value: 500
+evaluation_strategy:
+  desc: null
+  value: IntervalStrategy.NO
+fp16:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+fp16_full_eval:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+gradient_accumulation_steps:
+  desc: null
+  value: 1
+greater_is_better:
+  desc: null
+  value: null
+group_by_length:
+  desc: null
+  value: false
+ignore_data_skip:
+  desc: null
+  value: false
+label_names:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+learning_rate:
+  desc: null
+  value: 0.0003
+length_column_name:
+  desc: null
+  value: length
+line_by_line:
+  desc: null
+  value: false
+load_best_model_at_end:
+  desc: null
+  value: false
+local_rank:
+  desc: null
+  value: -1
+log_level:
+  desc: null
+  value: -1
+log_level_replica:
+  desc: null
+  value: -1
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: ./runs/Jul09_11-15-50_t1v-n-112df4a9-w-0
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 500
+logging_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+lr_scheduler_type:
+  desc: null
+  value: SchedulerType.LINEAR
+max_grad_norm:
+  desc: null
+  value: 1.0
+max_seq_length:
+  desc: null
+  value: 128
+max_steps:
+  desc: null
+  value: -1
+metric_for_best_model:
+  desc: null
+  value: null
+mlm_probability:
+  desc: null
+  value: 0.15
+model_name_or_path:
+  desc: null
+  value: null
+model_type:
+  desc: null
+  value: roberta
+mp_parameters:
+  desc: null
+  value: ''
+no_cuda:
+  desc: null
+  value: false
+num_train_epochs:
+  desc: null
+  value: 8.0
+output_dir:
+  desc: null
+  value: ./
+overwrite_cache:
+  desc: null
+  value: false
+overwrite_output_dir:
+  desc: null
+  value: true
+pad_to_max_length:
+  desc: null
+  value: false
+past_index:
+  desc: null
+  value: -1
+per_device_eval_batch_size:
+  desc: null
+  value: 4
+per_device_train_batch_size:
+  desc: null
+  value: 4
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+prediction_loss_only:
+  desc: null
+  value: false
+preprocessing_num_workers:
+  desc: null
+  value: null
+push_to_hub:
+  desc: null
+  value: false
+push_to_hub_model_id:
+  desc: null
+  value: ''
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: null
+remove_unused_columns:
+  desc: null
+  value: true
+report_to:
+  desc: null
+  value:
+  - tensorboard
+  - wandb
+resume_from_checkpoint:
+  desc: null
+  value: null
+run_name:
+  desc: null
+  value: ./
+save_on_each_node:
+  desc: null
+  value: false
+save_steps:
+  desc: null
+  value: 500
+save_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+save_total_limit:
+  desc: null
+  value: null
+seed:
+  desc: null
+  value: 42
+sharded_ddp:
+  desc: null
+  value: []
+skip_memory_metrics:
+  desc: null
+  value: true
+tokenizer_name:
+  desc: null
+  value: ./
+tpu_metrics_debug:
+  desc: null
+  value: false
+tpu_num_cores:
+  desc: null
+  value: null
+train_file:
+  desc: null
+  value: null
+train_ref_file:
+  desc: null
+  value: null
+use_fast_tokenizer:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+validation_file:
+  desc: null
+  value: null
+validation_ref_file:
+  desc: null
+  value: null
+validation_split_percentage:
+  desc: null
+  value: 5
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 1000
+weight_decay:
+  desc: null
+  value: 0.0

wandb/run-20210709_111555-275xd40l/files/output.log

@@ -0,0 +1,33 @@
+
+[11:15:56] - WARNING - __main__ - Unable to display metrics through TensorBoard because some package are not installed: No module named 'tensorflow'
+/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
+  warnings.warn(
+/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
+  warnings.warn(
+Epoch ... (1/8):   0%|                                                                                                                                                                | 0/8 [00:00<?, ?it/s]
+
+
+Epoch ... (1/8):  12%|███████████████████                                                                                                                                     | 1/8 [01:13<08:36, 73.75s/it]
+
+Epoch ... (1/8):  25%|██████████████████████████████████████                                                                                                                  | 2/8 [01:18<03:19, 33.23s/it]
+
+
+Epoch ... (1/8):  38%|█████████████████████████████████████████████████████████                                                                                               | 3/8 [01:23<01:41, 20.39s/it]
+Training...:  20%|██████████████████████████████▏                                                                                                                          | 28/142 [00:00<00:03, 37.68it/s]
+Training...:  51%|█████████████████████████████████████████████████████████████████████████████▌                                                                           | 72/142 [00:02<00:01, 36.09it/s]
+
+
+Step... (500 | Loss: 8.205772399902344, Acc: 0.0773010179400444):  50%|███████████████████████████████████████████████████▌                                                   | 4/8 [01:39<01:14, 18.65s/it]
+
+Step... (500 | Loss: 8.205772399902344, Acc: 0.0773010179400444):  62%|████████████████████████████████████████████████████████████████▍                                      | 5/8 [01:44<00:40, 13.57s/it]
+
+
+Step... (500 | Loss: 8.205772399902344, Acc: 0.0773010179400444):  75%|█████████████████████████████████████████████████████████████████████████████▎                         | 6/8 [01:49<00:21, 10.68s/it]
+
+Evaluating ...: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:00<00:00, 77.04it/s]
+Training...:   3%|████▎                                                                                                                                                     | 4/142 [00:00<00:03, 35.08it/s]
+Evaluating ...:  80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                              | 8/10 [00:00<00:00, 76.37it/s]
+
+
+
+Step... (1000 | Loss: 7.925442218780518, Acc: 0.09575153142213821): 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [02:05<00:00, 15.69s/it]

wandb/run-20210709_111555-275xd40l/files/requirements.txt

@@ -0,0 +1,96 @@
+absl-py==0.13.0
+aiohttp==3.7.4.post0
+async-timeout==3.0.1
+attrs==21.2.0
+backcall==0.2.0
+cachetools==4.2.2
+certifi==2021.5.30
+chardet==4.0.0
+chex==0.0.8
+click==8.0.1
+configparser==5.0.2
+cycler==0.10.0
+datasets==1.9.1.dev0
+decorator==5.0.9
+dill==0.3.4
+dm-tree==0.1.6
+docker-pycreds==0.4.0
+filelock==3.0.12
+flatbuffers==2.0
+flax==0.3.4
+fsspec==2021.6.1
+gitdb==4.0.7
+gitpython==3.1.18
+google-auth-oauthlib==0.4.4
+google-auth==1.32.1
+grpcio==1.38.1
+huggingface-hub==0.0.12
+idna==2.10
+ipython-genutils==0.2.0
+ipython==7.25.0
+jax==0.2.16
+jaxlib==0.1.68
+jedi==0.18.0
+joblib==1.0.1
+kiwisolver==1.3.1
+libtpu-nightly==0.1.dev20210615
+markdown==3.3.4
+matplotlib-inline==0.1.2
+matplotlib==3.4.2
+msgpack==1.0.2
+multidict==5.1.0
+multiprocess==0.70.12.2
+numpy==1.21.0
+oauthlib==3.1.1
+opt-einsum==3.3.0
+optax==0.0.9
+packaging==21.0
+pandas==1.3.0
+parso==0.8.2
+pathtools==0.1.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==8.3.1
+pip==20.0.2
+pkg-resources==0.0.0
+promise==2.3
+prompt-toolkit==3.0.19
+protobuf==3.17.3
+psutil==5.8.0
+ptyprocess==0.7.0
+pyarrow==4.0.1
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pygments==2.9.0
+pyparsing==2.4.7
+python-dateutil==2.8.1
+pytz==2021.1
+pyyaml==5.4.1
+regex==2021.7.6
+requests-oauthlib==1.3.0
+requests==2.25.1
+rsa==4.7.2
+sacremoses==0.0.45
+scipy==1.7.0
+sentry-sdk==1.3.0
+setuptools==44.0.0
+shortuuid==1.0.1
+six==1.16.0
+smmap==4.0.0
+subprocess32==3.5.4
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.5.0
+tokenizers==0.10.3
+toolz==0.11.1
+tqdm==4.61.2
+traitlets==5.0.5
+transformers==4.9.0.dev0
+typing-extensions==3.10.0.0
+urllib3==1.26.6
+wandb==0.10.33
+wcwidth==0.2.5
+werkzeug==2.0.1
+wheel==0.36.2
+xxhash==2.0.2
+yarl==1.6.3

wandb/run-20210709_111555-275xd40l/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+    "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
+    "python": "3.8.10",
+    "heartbeatAt": "2021-07-09T11:15:56.409747",
+    "startedAt": "2021-07-09T11:15:55.658458",
+    "docker": null,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [
+        "--output_dir=./",
+        "--model_type=roberta",
+        "--config_name=./",
+        "--tokenizer_name=./",
+        "--dataset_name=oscar",
+        "--dataset_config_name=unshuffled_deduplicated_als",
+        "--max_seq_length=128",
+        "--per_device_train_batch_size=4",
+        "--per_device_eval_batch_size=4",
+        "--learning_rate=3e-4",
+        "--warmup_steps=1000",
+        "--overwrite_output_dir",
+        "--num_train_epochs=8"
+    ],
+    "state": "running",
+    "program": "run_mlm_flax.py",
+    "codePath": "run_mlm_flax.py",
+    "git": {
+        "remote": "https://huggingface.co/flax-community/roberta-base-mr",
+        "commit": "faa64e8a79301832a56c78b2b293675039070bb0"
+    },
+    "email": "nipunsadvilkar@gmail.com",
+    "root": "/home/nipunsadvilkar/sample/roberta-base-mr",
+    "host": "t1v-n-112df4a9-w-0",
+    "username": "nipunsadvilkar",
+    "executable": "/home/nipunsadvilkar/roberta_mr_env/bin/python"
+}

wandb/run-20210709_111555-275xd40l/files/wandb-summary.json

@@ -0,0 +1 @@
+{}

wandb/run-20210709_111555-275xd40l/logs/debug-internal.log

@@ -0,0 +1,175 @@
+2021-07-09 11:15:56,246 INFO    MainThread:29289 [internal.py:wandb_internal():88] W&B internal server running at pid: 29289, started at: 2021-07-09 11:15:56.246443
+2021-07-09 11:15:56,248 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: check_version
+2021-07-09 11:15:56,248 INFO    WriterThread:29289 [datastore.py:open_for_write():80] open: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/run-275xd40l.wandb
+2021-07-09 11:15:56,249 DEBUG   SenderThread:29289 [sender.py:send():179] send: header
+2021-07-09 11:15:56,249 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: check_version
+2021-07-09 11:15:56,311 DEBUG   SenderThread:29289 [sender.py:send():179] send: run
+2021-07-09 11:15:56,407 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: run_start
+2021-07-09 11:15:56,408 INFO    SenderThread:29289 [dir_watcher.py:__init__():168] watching files in: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files
+2021-07-09 11:15:56,408 INFO    SenderThread:29289 [sender.py:_start_run_threads():716] run started: 275xd40l with start time 1625829355
+2021-07-09 11:15:56,408 DEBUG   SenderThread:29289 [sender.py:send():179] send: summary
+2021-07-09 11:15:56,409 INFO    SenderThread:29289 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-09 11:15:56,409 DEBUG   HandlerThread:29289 [meta.py:__init__():39] meta init
+2021-07-09 11:15:56,409 DEBUG   HandlerThread:29289 [meta.py:__init__():53] meta init done
+2021-07-09 11:15:56,409 DEBUG   HandlerThread:29289 [meta.py:probe():210] probe
+2021-07-09 11:15:56,410 DEBUG   HandlerThread:29289 [meta.py:_setup_git():200] setup git
+2021-07-09 11:15:56,426 DEBUG   HandlerThread:29289 [meta.py:_setup_git():207] setup git done
+2021-07-09 11:15:56,426 DEBUG   HandlerThread:29289 [meta.py:_save_pip():57] save pip
+2021-07-09 11:15:56,426 DEBUG   HandlerThread:29289 [meta.py:_save_pip():71] save pip done
+2021-07-09 11:15:56,426 DEBUG   HandlerThread:29289 [meta.py:probe():252] probe done
+2021-07-09 11:15:56,454 DEBUG   SenderThread:29289 [sender.py:send():179] send: files
+2021-07-09 11:15:56,455 INFO    SenderThread:29289 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now
+2021-07-09 11:15:56,459 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:15:56,460 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:15:56,485 DEBUG   SenderThread:29289 [sender.py:send():179] send: config
+2021-07-09 11:15:56,485 DEBUG   SenderThread:29289 [sender.py:send():179] send: config
+2021-07-09 11:15:56,485 DEBUG   SenderThread:29289 [sender.py:send():179] send: config
+2021-07-09 11:15:56,689 INFO    Thread-11 :29289 [upload_job.py:push():137] Uploaded file /tmp/tmp260my7l0wandb/rfatvpgl-wandb-metadata.json
+2021-07-09 11:15:57,409 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/wandb-summary.json
+2021-07-09 11:15:57,409 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/wandb-metadata.json
+2021-07-09 11:15:57,409 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:15:57,409 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/requirements.txt
+2021-07-09 11:15:59,409 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:16:03,410 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:16:05,412 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:16:11,521 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:16:11,521 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:16:24,485 DEBUG   SenderThread:29289 [sender.py:send():179] send: stats
+2021-07-09 11:16:26,562 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:16:26,562 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:16:27,421 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/config.yaml
+2021-07-09 11:16:41,590 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:16:41,590 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:16:54,566 DEBUG   SenderThread:29289 [sender.py:send():179] send: stats
+2021-07-09 11:16:56,619 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:16:56,619 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:17:11,651 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:17:11,652 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:17:13,440 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:15,441 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:17,442 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:19,443 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:21,444 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:23,444 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:24,641 DEBUG   SenderThread:29289 [sender.py:send():179] send: stats
+2021-07-09 11:17:25,445 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:26,687 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:17:26,688 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:17:27,446 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:29,447 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:33,449 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:41,453 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:41,727 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:17:41,727 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:17:43,454 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:45,455 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:47,456 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:49,457 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:51,458 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:53,459 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:54,712 DEBUG   SenderThread:29289 [sender.py:send():179] send: stats
+2021-07-09 11:17:55,459 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:56,763 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:17:56,763 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:17:58,461 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:17:59,461 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:18:04,463 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:18:06,464 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:18:08,465 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:18:08,711 DEBUG   SenderThread:29289 [sender.py:send():179] send: telemetry
+2021-07-09 11:18:08,711 DEBUG   SenderThread:29289 [sender.py:send():179] send: exit
+2021-07-09 11:18:08,711 INFO    SenderThread:29289 [sender.py:send_exit():287] handling exit code: 0
+2021-07-09 11:18:08,711 INFO    SenderThread:29289 [sender.py:send_exit():295] send defer
+2021-07-09 11:18:08,712 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:18:08,712 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:18:08,712 INFO    HandlerThread:29289 [handler.py:handle_request_defer():141] handle defer: 0
+2021-07-09 11:18:08,712 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:18:08,712 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:18:08,712 INFO    SenderThread:29289 [sender.py:send_request_defer():304] handle sender defer: 0
+2021-07-09 11:18:08,712 INFO    SenderThread:29289 [sender.py:transition_state():308] send defer: 1
+2021-07-09 11:18:08,712 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:18:08,712 INFO    HandlerThread:29289 [handler.py:handle_request_defer():141] handle defer: 1
+2021-07-09 11:18:08,745 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:18:08,745 INFO    SenderThread:29289 [sender.py:send_request_defer():304] handle sender defer: 1
+2021-07-09 11:18:08,745 INFO    SenderThread:29289 [sender.py:transition_state():308] send defer: 2
+2021-07-09 11:18:08,746 DEBUG   SenderThread:29289 [sender.py:send():179] send: stats
+2021-07-09 11:18:08,746 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:18:08,746 INFO    HandlerThread:29289 [handler.py:handle_request_defer():141] handle defer: 2
+2021-07-09 11:18:08,746 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:18:08,746 INFO    SenderThread:29289 [sender.py:send_request_defer():304] handle sender defer: 2
+2021-07-09 11:18:08,746 INFO    SenderThread:29289 [sender.py:transition_state():308] send defer: 3
+2021-07-09 11:18:08,746 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:18:08,746 INFO    HandlerThread:29289 [handler.py:handle_request_defer():141] handle defer: 3
+2021-07-09 11:18:08,747 DEBUG   SenderThread:29289 [sender.py:send():179] send: summary
+2021-07-09 11:18:08,747 INFO    SenderThread:29289 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-09 11:18:08,747 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:18:08,747 INFO    SenderThread:29289 [sender.py:send_request_defer():304] handle sender defer: 3
+2021-07-09 11:18:08,747 INFO    SenderThread:29289 [sender.py:transition_state():308] send defer: 4
+2021-07-09 11:18:08,747 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:18:08,747 INFO    HandlerThread:29289 [handler.py:handle_request_defer():141] handle defer: 4
+2021-07-09 11:18:08,748 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:18:08,748 INFO    SenderThread:29289 [sender.py:send_request_defer():304] handle sender defer: 4
+2021-07-09 11:18:08,811 INFO    SenderThread:29289 [sender.py:transition_state():308] send defer: 5
+2021-07-09 11:18:08,812 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:18:08,812 INFO    HandlerThread:29289 [handler.py:handle_request_defer():141] handle defer: 5
+2021-07-09 11:18:08,812 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:18:08,812 INFO    SenderThread:29289 [sender.py:send_request_defer():304] handle sender defer: 5
+2021-07-09 11:18:08,812 INFO    SenderThread:29289 [dir_watcher.py:finish():282] shutting down directory watcher
+2021-07-09 11:18:08,814 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:18:09,466 INFO    Thread-8  :29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/wandb-summary.json
+2021-07-09 11:18:09,466 INFO    SenderThread:29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/config.yaml
+2021-07-09 11:18:09,466 INFO    SenderThread:29289 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:18:09,467 INFO    SenderThread:29289 [dir_watcher.py:finish():312] scan: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files
+2021-07-09 11:18:09,467 INFO    SenderThread:29289 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/requirements.txt requirements.txt
+2021-07-09 11:18:09,467 INFO    SenderThread:29289 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log output.log
+2021-07-09 11:18:09,467 INFO    SenderThread:29289 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/wandb-metadata.json wandb-metadata.json
+2021-07-09 11:18:09,467 INFO    SenderThread:29289 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/config.yaml config.yaml
+2021-07-09 11:18:09,467 INFO    SenderThread:29289 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/wandb-summary.json wandb-summary.json
+2021-07-09 11:18:09,468 INFO    SenderThread:29289 [sender.py:transition_state():308] send defer: 6
+2021-07-09 11:18:09,468 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:18:09,471 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:18:09,471 INFO    HandlerThread:29289 [handler.py:handle_request_defer():141] handle defer: 6
+2021-07-09 11:18:09,473 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:18:09,473 INFO    SenderThread:29289 [sender.py:send_request_defer():304] handle sender defer: 6
+2021-07-09 11:18:09,474 INFO    SenderThread:29289 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-09 11:18:09,570 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:18:09,570 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:18:09,672 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:18:09,672 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:18:09,676 INFO    Thread-12 :29289 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/requirements.txt
+2021-07-09 11:18:09,678 INFO    Thread-14 :29289 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/config.yaml
+2021-07-09 11:18:09,688 INFO    Thread-15 :29289 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/wandb-summary.json
+2021-07-09 11:18:09,700 INFO    Thread-13 :29289 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/files/output.log
+2021-07-09 11:18:09,773 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:18:09,773 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:18:09,875 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:18:09,875 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:18:09,900 INFO    Thread-7  :29289 [sender.py:transition_state():308] send defer: 7
+2021-07-09 11:18:09,901 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:18:09,901 INFO    HandlerThread:29289 [handler.py:handle_request_defer():141] handle defer: 7
+2021-07-09 11:18:09,901 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:18:09,901 INFO    SenderThread:29289 [sender.py:send_request_defer():304] handle sender defer: 7
+2021-07-09 11:18:09,972 INFO    SenderThread:29289 [sender.py:transition_state():308] send defer: 8
+2021-07-09 11:18:09,973 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:18:09,973 INFO    HandlerThread:29289 [handler.py:handle_request_defer():141] handle defer: 8
+2021-07-09 11:18:09,973 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:18:09,973 INFO    SenderThread:29289 [sender.py:send_request_defer():304] handle sender defer: 8
+2021-07-09 11:18:09,973 INFO    SenderThread:29289 [sender.py:transition_state():308] send defer: 9
+2021-07-09 11:18:09,974 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:18:09,974 INFO    HandlerThread:29289 [handler.py:handle_request_defer():141] handle defer: 9
+2021-07-09 11:18:09,974 DEBUG   SenderThread:29289 [sender.py:send():179] send: final
+2021-07-09 11:18:09,974 DEBUG   SenderThread:29289 [sender.py:send():179] send: footer
+2021-07-09 11:18:09,974 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:18:09,974 INFO    SenderThread:29289 [sender.py:send_request_defer():304] handle sender defer: 9
+2021-07-09 11:18:09,977 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:18:09,977 DEBUG   SenderThread:29289 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:18:09,977 INFO    SenderThread:29289 [file_pusher.py:join():182] waiting for file pusher
+2021-07-09 11:18:09,978 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: get_summary
+2021-07-09 11:18:09,978 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: sampled_history
+2021-07-09 11:18:09,979 DEBUG   HandlerThread:29289 [handler.py:handle_request():124] handle_request: shutdown
+2021-07-09 11:18:09,979 INFO    HandlerThread:29289 [handler.py:finish():638] shutting down handler
+2021-07-09 11:18:10,974 INFO    WriterThread:29289 [datastore.py:close():288] close: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/run-275xd40l.wandb
+2021-07-09 11:18:10,977 INFO    SenderThread:29289 [sender.py:finish():945] shutting down sender
+2021-07-09 11:18:10,977 INFO    SenderThread:29289 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-09 11:18:10,977 INFO    SenderThread:29289 [file_pusher.py:join():182] waiting for file pusher
+2021-07-09 11:18:10,980 INFO    MainThread:29289 [internal.py:handle_exit():78] Internal process exited

wandb/run-20210709_111555-275xd40l/logs/debug.log

@@ -0,0 +1,88 @@
+2021-07-09 11:15:55,659 INFO    MainThread:28231 [wandb_setup.py:_flush():69] Unhandled environment var: WANDB_WATCH
+2021-07-09 11:15:55,660 INFO    MainThread:28231 [wandb_setup.py:_flush():69] setting env: {'project': 'hf-flax-robert-base-mr', 'entity': 'nipunsadvilkar'}
+2021-07-09 11:15:55,660 INFO    MainThread:28231 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-07-09 11:15:55,660 INFO    MainThread:28231 [wandb_init.py:_log_setup():337] Logging user logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/logs/debug.log
+2021-07-09 11:15:55,660 INFO    MainThread:28231 [wandb_init.py:_log_setup():338] Logging internal logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_111555-275xd40l/logs/debug-internal.log
+2021-07-09 11:15:55,660 INFO    MainThread:28231 [wandb_init.py:init():370] calling init triggers
+2021-07-09 11:15:55,660 INFO    MainThread:28231 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
+config: {}
+2021-07-09 11:15:55,660 INFO    MainThread:28231 [wandb_init.py:init():419] starting backend
+2021-07-09 11:15:55,660 INFO    MainThread:28231 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-07-09 11:15:55,687 INFO    MainThread:28231 [backend.py:ensure_launched():135] starting backend process...
+2021-07-09 11:15:55,713 INFO    MainThread:28231 [backend.py:ensure_launched():139] started backend process with pid: 29289
+2021-07-09 11:15:55,714 INFO    MainThread:28231 [wandb_init.py:init():424] backend started and connected
+2021-07-09 11:15:55,716 INFO    MainThread:28231 [wandb_init.py:init():472] updated telemetry
+2021-07-09 11:15:55,717 INFO    MainThread:28231 [wandb_init.py:init():491] communicating current version
+2021-07-09 11:15:56,309 INFO    MainThread:28231 [wandb_init.py:init():496] got version response 
+2021-07-09 11:15:56,310 INFO    MainThread:28231 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
+2021-07-09 11:15:56,406 INFO    MainThread:28231 [wandb_init.py:init():529] starting run threads in backend
+2021-07-09 11:15:56,456 INFO    MainThread:28231 [wandb_run.py:_console_start():1623] atexit reg
+2021-07-09 11:15:56,457 INFO    MainThread:28231 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
+2021-07-09 11:15:56,457 INFO    MainThread:28231 [wandb_run.py:_redirect():1502] Redirecting console.
+2021-07-09 11:15:56,459 INFO    MainThread:28231 [wandb_run.py:_redirect():1558] Redirects installed.
+2021-07-09 11:15:56,459 INFO    MainThread:28231 [wandb_init.py:init():554] run started, returning control to user process
+2021-07-09 11:15:56,460 INFO    MainThread:28231 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 4, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 8.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul09_11-15-50_t1v-n-112df4a9-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 500, 'save_total_limit': None, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
+2021-07-09 11:15:56,460 INFO    MainThread:28231 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'roberta', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
+2021-07-09 11:15:56,461 INFO    MainThread:28231 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': 'oscar', 'dataset_config_name': 'unshuffled_deduplicated_als', 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 128, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False}
+2021-07-09 11:18:06,643 INFO    MainThread:28231 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 0
+2021-07-09 11:18:06,644 INFO    MainThread:28231 [wandb_run.py:_restore():1565] restore
+2021-07-09 11:18:08,713 INFO    MainThread:28231 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 1200
+  total_bytes: 1200
+}
+
+2021-07-09 11:18:09,469 INFO    MainThread:28231 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 4
+}
+pusher_stats {
+  uploaded_bytes: 1200
+  total_bytes: 12360
+}
+
+2021-07-09 11:18:09,571 INFO    MainThread:28231 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12362
+  total_bytes: 12362
+}
+
+2021-07-09 11:18:09,673 INFO    MainThread:28231 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12362
+  total_bytes: 12362
+}
+
+2021-07-09 11:18:09,774 INFO    MainThread:28231 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12362
+  total_bytes: 12362
+}
+
+2021-07-09 11:18:09,876 INFO    MainThread:28231 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12362
+  total_bytes: 12362
+}
+
+2021-07-09 11:18:09,977 INFO    MainThread:28231 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true
+exit_result {
+}
+file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12362
+  total_bytes: 12362
+}
+
+2021-07-09 11:18:11,050 INFO    MainThread:28231 [wandb_run.py:_show_files():1937] logging synced files

wandb/run-20210709_115016-mgpt342u/files/config.yaml

@@ -0,0 +1,305 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.10.33
+    framework: huggingface
+    huggingface_version: 4.9.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.8.10
+    t:
+      1:
+      - 11
+      2:
+      - 11
+      4: 3.8.10
+      5: 0.10.33
+      6: 4.9.0.dev0
+      8:
+      - 5
+adafactor:
+  desc: null
+  value: false
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+cache_dir:
+  desc: null
+  value: null
+config_name:
+  desc: null
+  value: ./
+dataloader_drop_last:
+  desc: null
+  value: false
+dataloader_num_workers:
+  desc: null
+  value: 0
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataset_config_name:
+  desc: null
+  value: unshuffled_deduplicated_als
+dataset_name:
+  desc: null
+  value: oscar
+ddp_find_unused_parameters:
+  desc: null
+  value: null
+debug:
+  desc: null
+  value: []
+deepspeed:
+  desc: null
+  value: null
+disable_tqdm:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: false
+do_predict:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+dtype:
+  desc: null
+  value: float32
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_steps:
+  desc: null
+  value: 500
+evaluation_strategy:
+  desc: null
+  value: IntervalStrategy.NO
+fp16:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+fp16_full_eval:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+gradient_accumulation_steps:
+  desc: null
+  value: 1
+greater_is_better:
+  desc: null
+  value: null
+group_by_length:
+  desc: null
+  value: false
+ignore_data_skip:
+  desc: null
+  value: false
+label_names:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+learning_rate:
+  desc: null
+  value: 0.0003
+length_column_name:
+  desc: null
+  value: length
+line_by_line:
+  desc: null
+  value: false
+load_best_model_at_end:
+  desc: null
+  value: false
+local_rank:
+  desc: null
+  value: -1
+log_level:
+  desc: null
+  value: -1
+log_level_replica:
+  desc: null
+  value: -1
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: ./runs/Jul09_11-50-10_t1v-n-112df4a9-w-0
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 500
+logging_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+lr_scheduler_type:
+  desc: null
+  value: SchedulerType.LINEAR
+max_grad_norm:
+  desc: null
+  value: 1.0
+max_seq_length:
+  desc: null
+  value: 128
+max_steps:
+  desc: null
+  value: -1
+metric_for_best_model:
+  desc: null
+  value: null
+mlm_probability:
+  desc: null
+  value: 0.15
+model_name_or_path:
+  desc: null
+  value: null
+model_type:
+  desc: null
+  value: roberta
+mp_parameters:
+  desc: null
+  value: ''
+no_cuda:
+  desc: null
+  value: false
+num_train_epochs:
+  desc: null
+  value: 8.0
+output_dir:
+  desc: null
+  value: ./
+overwrite_cache:
+  desc: null
+  value: false
+overwrite_output_dir:
+  desc: null
+  value: true
+pad_to_max_length:
+  desc: null
+  value: false
+past_index:
+  desc: null
+  value: -1
+per_device_eval_batch_size:
+  desc: null
+  value: 4
+per_device_train_batch_size:
+  desc: null
+  value: 4
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+prediction_loss_only:
+  desc: null
+  value: false
+preprocessing_num_workers:
+  desc: null
+  value: null
+push_to_hub:
+  desc: null
+  value: false
+push_to_hub_model_id:
+  desc: null
+  value: ''
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: null
+remove_unused_columns:
+  desc: null
+  value: true
+report_to:
+  desc: null
+  value:
+  - tensorboard
+  - wandb
+resume_from_checkpoint:
+  desc: null
+  value: null
+run_name:
+  desc: null
+  value: ./
+save_on_each_node:
+  desc: null
+  value: false
+save_steps:
+  desc: null
+  value: 500
+save_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+save_total_limit:
+  desc: null
+  value: null
+seed:
+  desc: null
+  value: 42
+sharded_ddp:
+  desc: null
+  value: []
+skip_memory_metrics:
+  desc: null
+  value: true
+tokenizer_name:
+  desc: null
+  value: ./
+tpu_metrics_debug:
+  desc: null
+  value: false
+tpu_num_cores:
+  desc: null
+  value: null
+train_file:
+  desc: null
+  value: null
+train_ref_file:
+  desc: null
+  value: null
+use_fast_tokenizer:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+validation_file:
+  desc: null
+  value: null
+validation_ref_file:
+  desc: null
+  value: null
+validation_split_percentage:
+  desc: null
+  value: 5
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 1000
+weight_decay:
+  desc: null
+  value: 0.0

wandb/run-20210709_115016-mgpt342u/files/output.log

@@ -0,0 +1,34 @@
+
+[11:50:17] - WARNING - __main__ - Unable to display metrics through TensorBoard because some package are not installed: No module named 'tensorflow'
+/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
+  warnings.warn(
+/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
+  warnings.warn(
+Epoch ... (1/8):   0%|                                                                                                                                                                | 0/8 [00:00<?, ?it/s]
+
+
+Epoch ... (1/8):  12%|███████████████████                                                                                                                                     | 1/8 [01:17<09:01, 77.32s/it]
+
+
+Epoch ... (1/8):  25%|██████████████████████████████████████                                                                                                                  | 2/8 [01:22<03:28, 34.83s/it]
+
+Epoch ... (1/8):  38%|█████████████████████████████████████████████████████████                                                                                               | 3/8 [01:27<01:46, 21.26s/it]
+Training...:   8%|████████████▉                                                                                                                                            | 12/142 [00:00<00:03, 37.10it/s]
+
+Training...:  51%|█████████████████████████████████████████████████████████████████████████████▌                                                                           | 72/142 [00:02<00:02, 34.45it/s]
+
+
+Step... (500 | Loss: 8.205772399902344, Acc: 0.0773010179400444):  50%|███████████████████████████████████████████████████▌                                                   | 4/8 [01:43<01:17, 19.34s/it]
+
+
+Step... (500 | Loss: 8.205772399902344, Acc: 0.0773010179400444):  62%|████████████████████████████████████████████████████████████████▍                                      | 5/8 [01:48<00:42, 14.01s/it]
+
+Step... (500 | Loss: 8.205772399902344, Acc: 0.0773010179400444):  75%|█████████████████████████████████████████████████████████████████████████████▎                         | 6/8 [01:53<00:21, 10.96s/it]
+
+
+Step... (500 | Loss: 8.205772399902344, Acc: 0.0773010179400444):  88%|██████████████████████████████████████████████████████████████████████████████████████████▏            | 7/8 [01:58<00:09,  9.03s/it]
+Evaluating ...: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:00<00:00, 77.42it/s]
+Training...:   3%|████▎                                                                                                                                                     | 4/142 [00:00<00:03, 35.26it/s]
+
+
+Step... (1000 | Loss: 7.925442218780518, Acc: 0.09575153142213821): 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [02:09<00:00, 16.16s/it]

wandb/run-20210709_115016-mgpt342u/files/requirements.txt

@@ -0,0 +1,96 @@
+absl-py==0.13.0
+aiohttp==3.7.4.post0
+async-timeout==3.0.1
+attrs==21.2.0
+backcall==0.2.0
+cachetools==4.2.2
+certifi==2021.5.30
+chardet==4.0.0
+chex==0.0.8
+click==8.0.1
+configparser==5.0.2
+cycler==0.10.0
+datasets==1.9.1.dev0
+decorator==5.0.9
+dill==0.3.4
+dm-tree==0.1.6
+docker-pycreds==0.4.0
+filelock==3.0.12
+flatbuffers==2.0
+flax==0.3.4
+fsspec==2021.6.1
+gitdb==4.0.7
+gitpython==3.1.18
+google-auth-oauthlib==0.4.4
+google-auth==1.32.1
+grpcio==1.38.1
+huggingface-hub==0.0.12
+idna==2.10
+ipython-genutils==0.2.0
+ipython==7.25.0
+jax==0.2.16
+jaxlib==0.1.68
+jedi==0.18.0
+joblib==1.0.1
+kiwisolver==1.3.1
+libtpu-nightly==0.1.dev20210615
+markdown==3.3.4
+matplotlib-inline==0.1.2
+matplotlib==3.4.2
+msgpack==1.0.2
+multidict==5.1.0
+multiprocess==0.70.12.2
+numpy==1.21.0
+oauthlib==3.1.1
+opt-einsum==3.3.0
+optax==0.0.9
+packaging==21.0
+pandas==1.3.0
+parso==0.8.2
+pathtools==0.1.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==8.3.1
+pip==20.0.2
+pkg-resources==0.0.0
+promise==2.3
+prompt-toolkit==3.0.19
+protobuf==3.17.3
+psutil==5.8.0
+ptyprocess==0.7.0
+pyarrow==4.0.1
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pygments==2.9.0
+pyparsing==2.4.7
+python-dateutil==2.8.1
+pytz==2021.1
+pyyaml==5.4.1
+regex==2021.7.6
+requests-oauthlib==1.3.0
+requests==2.25.1
+rsa==4.7.2
+sacremoses==0.0.45
+scipy==1.7.0
+sentry-sdk==1.3.0
+setuptools==44.0.0
+shortuuid==1.0.1
+six==1.16.0
+smmap==4.0.0
+subprocess32==3.5.4
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.5.0
+tokenizers==0.10.3
+toolz==0.11.1
+tqdm==4.61.2
+traitlets==5.0.5
+transformers==4.9.0.dev0
+typing-extensions==3.10.0.0
+urllib3==1.26.6
+wandb==0.10.33
+wcwidth==0.2.5
+werkzeug==2.0.1
+wheel==0.36.2
+xxhash==2.0.2
+yarl==1.6.3

wandb/run-20210709_115016-mgpt342u/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+    "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
+    "python": "3.8.10",
+    "heartbeatAt": "2021-07-09T11:50:17.221650",
+    "startedAt": "2021-07-09T11:50:16.489745",
+    "docker": null,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [
+        "--output_dir=./",
+        "--model_type=roberta",
+        "--config_name=./",
+        "--tokenizer_name=./",
+        "--dataset_name=oscar",
+        "--dataset_config_name=unshuffled_deduplicated_als",
+        "--max_seq_length=128",
+        "--per_device_train_batch_size=4",
+        "--per_device_eval_batch_size=4",
+        "--learning_rate=3e-4",
+        "--warmup_steps=1000",
+        "--overwrite_output_dir",
+        "--num_train_epochs=8"
+    ],
+    "state": "running",
+    "program": "run_mlm_flax.py",
+    "codePath": "run_mlm_flax.py",
+    "git": {
+        "remote": "https://huggingface.co/flax-community/roberta-base-mr",
+        "commit": "faa64e8a79301832a56c78b2b293675039070bb0"
+    },
+    "email": "nipunsadvilkar@gmail.com",
+    "root": "/home/nipunsadvilkar/sample/roberta-base-mr",
+    "host": "t1v-n-112df4a9-w-0",
+    "username": "nipunsadvilkar",
+    "executable": "/home/nipunsadvilkar/roberta_mr_env/bin/python"
+}

wandb/run-20210709_115016-mgpt342u/files/wandb-summary.json

@@ -0,0 +1 @@
+{}

wandb/run-20210709_115016-mgpt342u/logs/debug-internal.log

@@ -0,0 +1,177 @@
+2021-07-09 11:50:17,074 INFO    MainThread:30716 [internal.py:wandb_internal():88] W&B internal server running at pid: 30716, started at: 2021-07-09 11:50:17.074054
+2021-07-09 11:50:17,076 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: check_version
+2021-07-09 11:50:17,076 INFO    WriterThread:30716 [datastore.py:open_for_write():80] open: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/run-mgpt342u.wandb
+2021-07-09 11:50:17,077 DEBUG   SenderThread:30716 [sender.py:send():179] send: header
+2021-07-09 11:50:17,077 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: check_version
+2021-07-09 11:50:17,138 DEBUG   SenderThread:30716 [sender.py:send():179] send: run
+2021-07-09 11:50:17,219 INFO    SenderThread:30716 [dir_watcher.py:__init__():168] watching files in: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files
+2021-07-09 11:50:17,219 INFO    SenderThread:30716 [sender.py:_start_run_threads():716] run started: mgpt342u with start time 1625831416
+2021-07-09 11:50:17,219 DEBUG   SenderThread:30716 [sender.py:send():179] send: summary
+2021-07-09 11:50:17,219 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: run_start
+2021-07-09 11:50:17,221 INFO    SenderThread:30716 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-09 11:50:17,221 DEBUG   HandlerThread:30716 [meta.py:__init__():39] meta init
+2021-07-09 11:50:17,221 DEBUG   HandlerThread:30716 [meta.py:__init__():53] meta init done
+2021-07-09 11:50:17,221 DEBUG   HandlerThread:30716 [meta.py:probe():210] probe
+2021-07-09 11:50:17,222 DEBUG   HandlerThread:30716 [meta.py:_setup_git():200] setup git
+2021-07-09 11:50:17,238 DEBUG   HandlerThread:30716 [meta.py:_setup_git():207] setup git done
+2021-07-09 11:50:17,238 DEBUG   HandlerThread:30716 [meta.py:_save_pip():57] save pip
+2021-07-09 11:50:17,239 DEBUG   HandlerThread:30716 [meta.py:_save_pip():71] save pip done
+2021-07-09 11:50:17,239 DEBUG   HandlerThread:30716 [meta.py:probe():252] probe done
+2021-07-09 11:50:17,267 DEBUG   SenderThread:30716 [sender.py:send():179] send: files
+2021-07-09 11:50:17,267 INFO    SenderThread:30716 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now
+2021-07-09 11:50:17,271 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:50:17,271 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:50:17,297 DEBUG   SenderThread:30716 [sender.py:send():179] send: config
+2021-07-09 11:50:17,297 DEBUG   SenderThread:30716 [sender.py:send():179] send: config
+2021-07-09 11:50:17,297 DEBUG   SenderThread:30716 [sender.py:send():179] send: config
+2021-07-09 11:50:17,553 INFO    Thread-11 :30716 [upload_job.py:push():137] Uploaded file /tmp/tmpt1i58m5_wandb/1ivr8ckt-wandb-metadata.json
+2021-07-09 11:50:18,220 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:50:18,221 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/wandb-metadata.json
+2021-07-09 11:50:18,221 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/requirements.txt
+2021-07-09 11:50:18,221 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/wandb-summary.json
+2021-07-09 11:50:20,221 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:50:24,223 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:50:26,224 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:50:32,314 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:50:32,315 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:50:45,291 DEBUG   SenderThread:30716 [sender.py:send():179] send: stats
+2021-07-09 11:50:47,357 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:50:47,358 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:50:48,234 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/config.yaml
+2021-07-09 11:51:02,431 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:51:02,431 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:51:15,372 DEBUG   SenderThread:30716 [sender.py:send():179] send: stats
+2021-07-09 11:51:17,464 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:51:17,464 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:51:32,530 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:51:32,530 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:51:38,258 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:51:40,259 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:51:42,260 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:51:44,261 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:51:45,445 DEBUG   SenderThread:30716 [sender.py:send():179] send: stats
+2021-07-09 11:51:46,262 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:51:47,561 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:51:47,561 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:51:48,263 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:51:50,264 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:51:52,264 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:51:54,266 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:51:56,267 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:00,269 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:02,619 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:52:02,620 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:52:06,272 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:08,273 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:10,274 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:12,275 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:14,276 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:15,519 DEBUG   SenderThread:30716 [sender.py:send():179] send: stats
+2021-07-09 11:52:16,277 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:17,653 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 11:52:17,653 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 11:52:18,278 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:20,279 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:22,280 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:24,281 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:30,284 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:32,285 INFO    Thread-8  :30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:33,545 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:52:33,546 DEBUG   SenderThread:30716 [sender.py:send():179] send: telemetry
+2021-07-09 11:52:33,546 DEBUG   SenderThread:30716 [sender.py:send():179] send: exit
+2021-07-09 11:52:33,546 INFO    SenderThread:30716 [sender.py:send_exit():287] handling exit code: 0
+2021-07-09 11:52:33,546 INFO    SenderThread:30716 [sender.py:send_exit():295] send defer
+2021-07-09 11:52:33,546 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:52:33,547 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:52:33,547 INFO    HandlerThread:30716 [handler.py:handle_request_defer():141] handle defer: 0
+2021-07-09 11:52:33,547 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:52:33,547 INFO    SenderThread:30716 [sender.py:send_request_defer():304] handle sender defer: 0
+2021-07-09 11:52:33,547 INFO    SenderThread:30716 [sender.py:transition_state():308] send defer: 1
+2021-07-09 11:52:33,547 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:52:33,547 INFO    HandlerThread:30716 [handler.py:handle_request_defer():141] handle defer: 1
+2021-07-09 11:52:33,560 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:52:33,561 INFO    SenderThread:30716 [sender.py:send_request_defer():304] handle sender defer: 1
+2021-07-09 11:52:33,561 INFO    SenderThread:30716 [sender.py:transition_state():308] send defer: 2
+2021-07-09 11:52:33,561 DEBUG   SenderThread:30716 [sender.py:send():179] send: stats
+2021-07-09 11:52:33,561 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:52:33,561 INFO    HandlerThread:30716 [handler.py:handle_request_defer():141] handle defer: 2
+2021-07-09 11:52:33,562 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:52:33,562 INFO    SenderThread:30716 [sender.py:send_request_defer():304] handle sender defer: 2
+2021-07-09 11:52:33,562 INFO    SenderThread:30716 [sender.py:transition_state():308] send defer: 3
+2021-07-09 11:52:33,562 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:52:33,562 INFO    HandlerThread:30716 [handler.py:handle_request_defer():141] handle defer: 3
+2021-07-09 11:52:33,562 DEBUG   SenderThread:30716 [sender.py:send():179] send: summary
+2021-07-09 11:52:33,563 INFO    SenderThread:30716 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-09 11:52:33,563 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:52:33,563 INFO    SenderThread:30716 [sender.py:send_request_defer():304] handle sender defer: 3
+2021-07-09 11:52:33,563 INFO    SenderThread:30716 [sender.py:transition_state():308] send defer: 4
+2021-07-09 11:52:33,563 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:52:33,563 INFO    HandlerThread:30716 [handler.py:handle_request_defer():141] handle defer: 4
+2021-07-09 11:52:33,563 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:52:33,563 INFO    SenderThread:30716 [sender.py:send_request_defer():304] handle sender defer: 4
+2021-07-09 11:52:33,629 INFO    SenderThread:30716 [sender.py:transition_state():308] send defer: 5
+2021-07-09 11:52:33,629 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:52:33,630 INFO    HandlerThread:30716 [handler.py:handle_request_defer():141] handle defer: 5
+2021-07-09 11:52:33,630 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:52:33,630 INFO    SenderThread:30716 [sender.py:send_request_defer():304] handle sender defer: 5
+2021-07-09 11:52:33,630 INFO    SenderThread:30716 [dir_watcher.py:finish():282] shutting down directory watcher
+2021-07-09 11:52:33,648 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:52:34,286 INFO    SenderThread:30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:34,286 INFO    SenderThread:30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/config.yaml
+2021-07-09 11:52:34,286 INFO    SenderThread:30716 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/wandb-summary.json
+2021-07-09 11:52:34,287 INFO    SenderThread:30716 [dir_watcher.py:finish():312] scan: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files
+2021-07-09 11:52:34,287 INFO    SenderThread:30716 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/requirements.txt requirements.txt
+2021-07-09 11:52:34,287 INFO    SenderThread:30716 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log output.log
+2021-07-09 11:52:34,287 INFO    SenderThread:30716 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/wandb-metadata.json wandb-metadata.json
+2021-07-09 11:52:34,287 INFO    SenderThread:30716 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/config.yaml config.yaml
+2021-07-09 11:52:34,288 INFO    SenderThread:30716 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/wandb-summary.json wandb-summary.json
+2021-07-09 11:52:34,288 INFO    SenderThread:30716 [sender.py:transition_state():308] send defer: 6
+2021-07-09 11:52:34,288 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:52:34,293 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:52:34,293 INFO    HandlerThread:30716 [handler.py:handle_request_defer():141] handle defer: 6
+2021-07-09 11:52:34,294 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:52:34,294 INFO    SenderThread:30716 [sender.py:send_request_defer():304] handle sender defer: 6
+2021-07-09 11:52:34,294 INFO    SenderThread:30716 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-09 11:52:34,390 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:52:34,390 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:52:34,492 INFO    Thread-15 :30716 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/wandb-summary.json
+2021-07-09 11:52:34,492 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:52:34,493 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:52:34,497 INFO    Thread-12 :30716 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/requirements.txt
+2021-07-09 11:52:34,536 INFO    Thread-13 :30716 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/output.log
+2021-07-09 11:52:34,563 INFO    Thread-14 :30716 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/files/config.yaml
+2021-07-09 11:52:34,594 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:52:34,595 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:52:34,696 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:52:34,697 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:52:34,764 INFO    Thread-7  :30716 [sender.py:transition_state():308] send defer: 7
+2021-07-09 11:52:34,765 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:52:34,765 INFO    HandlerThread:30716 [handler.py:handle_request_defer():141] handle defer: 7
+2021-07-09 11:52:34,765 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:52:34,765 INFO    SenderThread:30716 [sender.py:send_request_defer():304] handle sender defer: 7
+2021-07-09 11:52:34,798 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:52:34,838 INFO    SenderThread:30716 [sender.py:transition_state():308] send defer: 8
+2021-07-09 11:52:34,838 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:52:34,839 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:52:34,839 INFO    HandlerThread:30716 [handler.py:handle_request_defer():141] handle defer: 8
+2021-07-09 11:52:34,839 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:52:34,839 INFO    SenderThread:30716 [sender.py:send_request_defer():304] handle sender defer: 8
+2021-07-09 11:52:34,839 INFO    SenderThread:30716 [sender.py:transition_state():308] send defer: 9
+2021-07-09 11:52:34,840 DEBUG   SenderThread:30716 [sender.py:send():179] send: final
+2021-07-09 11:52:34,840 DEBUG   SenderThread:30716 [sender.py:send():179] send: footer
+2021-07-09 11:52:34,840 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 11:52:34,840 INFO    HandlerThread:30716 [handler.py:handle_request_defer():141] handle defer: 9
+2021-07-09 11:52:34,840 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: defer
+2021-07-09 11:52:34,840 INFO    SenderThread:30716 [sender.py:send_request_defer():304] handle sender defer: 9
+2021-07-09 11:52:34,940 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 11:52:34,940 DEBUG   SenderThread:30716 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 11:52:34,941 INFO    SenderThread:30716 [file_pusher.py:join():182] waiting for file pusher
+2021-07-09 11:52:34,942 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: get_summary
+2021-07-09 11:52:34,943 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: sampled_history
+2021-07-09 11:52:34,943 DEBUG   HandlerThread:30716 [handler.py:handle_request():124] handle_request: shutdown
+2021-07-09 11:52:34,943 INFO    HandlerThread:30716 [handler.py:finish():638] shutting down handler
+2021-07-09 11:52:35,840 INFO    WriterThread:30716 [datastore.py:close():288] close: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/run-mgpt342u.wandb
+2021-07-09 11:52:35,941 INFO    SenderThread:30716 [sender.py:finish():945] shutting down sender
+2021-07-09 11:52:35,941 INFO    SenderThread:30716 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-09 11:52:35,941 INFO    SenderThread:30716 [file_pusher.py:join():182] waiting for file pusher
+2021-07-09 11:52:35,944 INFO    MainThread:30716 [internal.py:handle_exit():78] Internal process exited

wandb/run-20210709_115016-mgpt342u/logs/debug.log

@@ -0,0 +1,96 @@
+2021-07-09 11:50:16,491 INFO    MainThread:29662 [wandb_setup.py:_flush():69] Unhandled environment var: WANDB_WATCH
+2021-07-09 11:50:16,491 INFO    MainThread:29662 [wandb_setup.py:_flush():69] setting env: {'project': 'hf-flax-robert-base-mr', 'entity': 'nipunsadvilkar'}
+2021-07-09 11:50:16,491 INFO    MainThread:29662 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-07-09 11:50:16,491 INFO    MainThread:29662 [wandb_init.py:_log_setup():337] Logging user logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/logs/debug.log
+2021-07-09 11:50:16,491 INFO    MainThread:29662 [wandb_init.py:_log_setup():338] Logging internal logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_115016-mgpt342u/logs/debug-internal.log
+2021-07-09 11:50:16,491 INFO    MainThread:29662 [wandb_init.py:init():370] calling init triggers
+2021-07-09 11:50:16,491 INFO    MainThread:29662 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
+config: {}
+2021-07-09 11:50:16,491 INFO    MainThread:29662 [wandb_init.py:init():419] starting backend
+2021-07-09 11:50:16,491 INFO    MainThread:29662 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-07-09 11:50:16,516 INFO    MainThread:29662 [backend.py:ensure_launched():135] starting backend process...
+2021-07-09 11:50:16,540 INFO    MainThread:29662 [backend.py:ensure_launched():139] started backend process with pid: 30716
+2021-07-09 11:50:16,542 INFO    MainThread:29662 [wandb_init.py:init():424] backend started and connected
+2021-07-09 11:50:16,545 INFO    MainThread:29662 [wandb_init.py:init():472] updated telemetry
+2021-07-09 11:50:16,545 INFO    MainThread:29662 [wandb_init.py:init():491] communicating current version
+2021-07-09 11:50:17,137 INFO    MainThread:29662 [wandb_init.py:init():496] got version response 
+2021-07-09 11:50:17,137 INFO    MainThread:29662 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
+2021-07-09 11:50:17,218 INFO    MainThread:29662 [wandb_init.py:init():529] starting run threads in backend
+2021-07-09 11:50:17,269 INFO    MainThread:29662 [wandb_run.py:_console_start():1623] atexit reg
+2021-07-09 11:50:17,269 INFO    MainThread:29662 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
+2021-07-09 11:50:17,270 INFO    MainThread:29662 [wandb_run.py:_redirect():1502] Redirecting console.
+2021-07-09 11:50:17,271 INFO    MainThread:29662 [wandb_run.py:_redirect():1558] Redirects installed.
+2021-07-09 11:50:17,272 INFO    MainThread:29662 [wandb_init.py:init():554] run started, returning control to user process
+2021-07-09 11:50:17,272 INFO    MainThread:29662 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 4, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 8.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul09_11-50-10_t1v-n-112df4a9-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 500, 'save_total_limit': None, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
+2021-07-09 11:50:17,273 INFO    MainThread:29662 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'roberta', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
+2021-07-09 11:50:17,274 INFO    MainThread:29662 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': 'oscar', 'dataset_config_name': 'unshuffled_deduplicated_als', 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 128, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False}
+2021-07-09 11:52:31,413 INFO    MainThread:29662 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 0
+2021-07-09 11:52:31,414 INFO    MainThread:29662 [wandb_run.py:_restore():1565] restore
+2021-07-09 11:52:33,547 INFO    MainThread:29662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 1200
+  total_bytes: 1200
+}
+
+2021-07-09 11:52:34,289 INFO    MainThread:29662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 2
+}
+pusher_stats {
+  uploaded_bytes: 1200
+  total_bytes: 2856
+}
+
+2021-07-09 11:52:34,391 INFO    MainThread:29662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12265
+  total_bytes: 12265
+}
+
+2021-07-09 11:52:34,494 INFO    MainThread:29662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12265
+  total_bytes: 12265
+}
+
+2021-07-09 11:52:34,596 INFO    MainThread:29662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12265
+  total_bytes: 12265
+}
+
+2021-07-09 11:52:34,698 INFO    MainThread:29662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12265
+  total_bytes: 12265
+}
+
+2021-07-09 11:52:34,839 INFO    MainThread:29662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12265
+  total_bytes: 12265
+}
+
+2021-07-09 11:52:34,941 INFO    MainThread:29662 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true
+exit_result {
+}
+file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12265
+  total_bytes: 12265
+}
+
+2021-07-09 11:52:36,012 INFO    MainThread:29662 [wandb_run.py:_show_files():1937] logging synced files

wandb/run-20210709_122021-2porvgvv/files/config.yaml

@@ -0,0 +1,305 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.10.33
+    framework: huggingface
+    huggingface_version: 4.9.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.8.10
+    t:
+      1:
+      - 11
+      2:
+      - 11
+      4: 3.8.10
+      5: 0.10.33
+      6: 4.9.0.dev0
+      8:
+      - 5
+adafactor:
+  desc: null
+  value: false
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+cache_dir:
+  desc: null
+  value: null
+config_name:
+  desc: null
+  value: ./
+dataloader_drop_last:
+  desc: null
+  value: false
+dataloader_num_workers:
+  desc: null
+  value: 0
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataset_config_name:
+  desc: null
+  value: unshuffled_deduplicated_als
+dataset_name:
+  desc: null
+  value: oscar
+ddp_find_unused_parameters:
+  desc: null
+  value: null
+debug:
+  desc: null
+  value: []
+deepspeed:
+  desc: null
+  value: null
+disable_tqdm:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: false
+do_predict:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+dtype:
+  desc: null
+  value: float32
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_steps:
+  desc: null
+  value: 500
+evaluation_strategy:
+  desc: null
+  value: IntervalStrategy.NO
+fp16:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+fp16_full_eval:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+gradient_accumulation_steps:
+  desc: null
+  value: 1
+greater_is_better:
+  desc: null
+  value: null
+group_by_length:
+  desc: null
+  value: false
+ignore_data_skip:
+  desc: null
+  value: false
+label_names:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+learning_rate:
+  desc: null
+  value: 0.0003
+length_column_name:
+  desc: null
+  value: length
+line_by_line:
+  desc: null
+  value: false
+load_best_model_at_end:
+  desc: null
+  value: false
+local_rank:
+  desc: null
+  value: -1
+log_level:
+  desc: null
+  value: -1
+log_level_replica:
+  desc: null
+  value: -1
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: ./runs/Jul09_12-20-15_t1v-n-112df4a9-w-0
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 500
+logging_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+lr_scheduler_type:
+  desc: null
+  value: SchedulerType.LINEAR
+max_grad_norm:
+  desc: null
+  value: 1.0
+max_seq_length:
+  desc: null
+  value: 128
+max_steps:
+  desc: null
+  value: -1
+metric_for_best_model:
+  desc: null
+  value: null
+mlm_probability:
+  desc: null
+  value: 0.15
+model_name_or_path:
+  desc: null
+  value: null
+model_type:
+  desc: null
+  value: roberta
+mp_parameters:
+  desc: null
+  value: ''
+no_cuda:
+  desc: null
+  value: false
+num_train_epochs:
+  desc: null
+  value: 8.0
+output_dir:
+  desc: null
+  value: ./
+overwrite_cache:
+  desc: null
+  value: false
+overwrite_output_dir:
+  desc: null
+  value: true
+pad_to_max_length:
+  desc: null
+  value: false
+past_index:
+  desc: null
+  value: -1
+per_device_eval_batch_size:
+  desc: null
+  value: 4
+per_device_train_batch_size:
+  desc: null
+  value: 4
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+prediction_loss_only:
+  desc: null
+  value: false
+preprocessing_num_workers:
+  desc: null
+  value: null
+push_to_hub:
+  desc: null
+  value: false
+push_to_hub_model_id:
+  desc: null
+  value: ''
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: null
+remove_unused_columns:
+  desc: null
+  value: true
+report_to:
+  desc: null
+  value:
+  - tensorboard
+  - wandb
+resume_from_checkpoint:
+  desc: null
+  value: null
+run_name:
+  desc: null
+  value: ./
+save_on_each_node:
+  desc: null
+  value: false
+save_steps:
+  desc: null
+  value: 500
+save_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+save_total_limit:
+  desc: null
+  value: null
+seed:
+  desc: null
+  value: 42
+sharded_ddp:
+  desc: null
+  value: []
+skip_memory_metrics:
+  desc: null
+  value: true
+tokenizer_name:
+  desc: null
+  value: ./
+tpu_metrics_debug:
+  desc: null
+  value: false
+tpu_num_cores:
+  desc: null
+  value: null
+train_file:
+  desc: null
+  value: null
+train_ref_file:
+  desc: null
+  value: null
+use_fast_tokenizer:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+validation_file:
+  desc: null
+  value: null
+validation_ref_file:
+  desc: null
+  value: null
+validation_split_percentage:
+  desc: null
+  value: 5
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 1000
+weight_decay:
+  desc: null
+  value: 0.0

wandb/run-20210709_122021-2porvgvv/files/output.log

@@ -0,0 +1,18 @@
+
+[12:20:22] - WARNING - __main__ - Unable to display metrics through TensorBoard because some package are not installed: No module named 'tensorflow'
+/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
+  warnings.warn(
+/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
+  warnings.warn(
+Epoch ... (1/8):   0%|                                                                                                                                                                | 0/8 [00:00<?, ?it/s]
+
+
+Epoch ... (1/8):  12%|███████████████████                                                                                                                                     | 1/8 [01:18<09:06, 78.12s/it]
+
+
+Epoch ... (1/8):  25%|██████████████████████████████████████                                                                                                                  | 2/8 [01:22<03:30, 35.03s/it]
+
+
+Epoch ... (1/8):  38%|█████████████████████████████████████████████████████████                                                                                               | 3/8 [01:27<01:46, 21.23s/it]
+
+Epoch ... (1/8):  38%|█████████████████████████████████████████████████████████                                                                                               | 3/8 [01:31<02:31, 30.34s/it]

wandb/run-20210709_122021-2porvgvv/files/requirements.txt

@@ -0,0 +1,96 @@
+absl-py==0.13.0
+aiohttp==3.7.4.post0
+async-timeout==3.0.1
+attrs==21.2.0
+backcall==0.2.0
+cachetools==4.2.2
+certifi==2021.5.30
+chardet==4.0.0
+chex==0.0.8
+click==8.0.1
+configparser==5.0.2
+cycler==0.10.0
+datasets==1.9.1.dev0
+decorator==5.0.9
+dill==0.3.4
+dm-tree==0.1.6
+docker-pycreds==0.4.0
+filelock==3.0.12
+flatbuffers==2.0
+flax==0.3.4
+fsspec==2021.6.1
+gitdb==4.0.7
+gitpython==3.1.18
+google-auth-oauthlib==0.4.4
+google-auth==1.32.1
+grpcio==1.38.1
+huggingface-hub==0.0.12
+idna==2.10
+ipython-genutils==0.2.0
+ipython==7.25.0
+jax==0.2.16
+jaxlib==0.1.68
+jedi==0.18.0
+joblib==1.0.1
+kiwisolver==1.3.1
+libtpu-nightly==0.1.dev20210615
+markdown==3.3.4
+matplotlib-inline==0.1.2
+matplotlib==3.4.2
+msgpack==1.0.2
+multidict==5.1.0
+multiprocess==0.70.12.2
+numpy==1.21.0
+oauthlib==3.1.1
+opt-einsum==3.3.0
+optax==0.0.9
+packaging==21.0
+pandas==1.3.0
+parso==0.8.2
+pathtools==0.1.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==8.3.1
+pip==20.0.2
+pkg-resources==0.0.0
+promise==2.3
+prompt-toolkit==3.0.19
+protobuf==3.17.3
+psutil==5.8.0
+ptyprocess==0.7.0
+pyarrow==4.0.1
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pygments==2.9.0
+pyparsing==2.4.7
+python-dateutil==2.8.1
+pytz==2021.1
+pyyaml==5.4.1
+regex==2021.7.6
+requests-oauthlib==1.3.0
+requests==2.25.1
+rsa==4.7.2
+sacremoses==0.0.45
+scipy==1.7.0
+sentry-sdk==1.3.0
+setuptools==44.0.0
+shortuuid==1.0.1
+six==1.16.0
+smmap==4.0.0
+subprocess32==3.5.4
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.5.0
+tokenizers==0.10.3
+toolz==0.11.1
+tqdm==4.61.2
+traitlets==5.0.5
+transformers==4.9.0.dev0
+typing-extensions==3.10.0.0
+urllib3==1.26.6
+wandb==0.10.33
+wcwidth==0.2.5
+werkzeug==2.0.1
+wheel==0.36.2
+xxhash==2.0.2
+yarl==1.6.3

wandb/run-20210709_122021-2porvgvv/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+    "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
+    "python": "3.8.10",
+    "heartbeatAt": "2021-07-09T12:20:22.116773",
+    "startedAt": "2021-07-09T12:20:21.341845",
+    "docker": null,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [
+        "--output_dir=./",
+        "--model_type=roberta",
+        "--config_name=./",
+        "--tokenizer_name=./",
+        "--dataset_name=oscar",
+        "--dataset_config_name=unshuffled_deduplicated_als",
+        "--max_seq_length=128",
+        "--per_device_train_batch_size=4",
+        "--per_device_eval_batch_size=4",
+        "--learning_rate=3e-4",
+        "--warmup_steps=1000",
+        "--overwrite_output_dir",
+        "--num_train_epochs=8"
+    ],
+    "state": "running",
+    "program": "run_mlm_flax.py",
+    "codePath": "run_mlm_flax.py",
+    "git": {
+        "remote": "https://huggingface.co/flax-community/roberta-base-mr",
+        "commit": "faa64e8a79301832a56c78b2b293675039070bb0"
+    },
+    "email": "nipunsadvilkar@gmail.com",
+    "root": "/home/nipunsadvilkar/sample/roberta-base-mr",
+    "host": "t1v-n-112df4a9-w-0",
+    "username": "nipunsadvilkar",
+    "executable": "/home/nipunsadvilkar/roberta_mr_env/bin/python"
+}

wandb/run-20210709_122021-2porvgvv/files/wandb-summary.json

@@ -0,0 +1 @@
+{}

wandb/run-20210709_122021-2porvgvv/logs/debug-internal.log

@@ -0,0 +1,156 @@
+2021-07-09 12:20:21,976 INFO    MainThread:32095 [internal.py:wandb_internal():88] W&B internal server running at pid: 32095, started at: 2021-07-09 12:20:21.975682
+2021-07-09 12:20:21,978 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: check_version
+2021-07-09 12:20:21,978 INFO    WriterThread:32095 [datastore.py:open_for_write():80] open: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/run-2porvgvv.wandb
+2021-07-09 12:20:21,979 DEBUG   SenderThread:32095 [sender.py:send():179] send: header
+2021-07-09 12:20:21,979 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: check_version
+2021-07-09 12:20:22,045 DEBUG   SenderThread:32095 [sender.py:send():179] send: run
+2021-07-09 12:20:22,114 INFO    SenderThread:32095 [dir_watcher.py:__init__():168] watching files in: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files
+2021-07-09 12:20:22,114 INFO    SenderThread:32095 [sender.py:_start_run_threads():716] run started: 2porvgvv with start time 1625833221
+2021-07-09 12:20:22,114 DEBUG   SenderThread:32095 [sender.py:send():179] send: summary
+2021-07-09 12:20:22,114 INFO    SenderThread:32095 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-09 12:20:22,115 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: run_start
+2021-07-09 12:20:22,116 DEBUG   HandlerThread:32095 [meta.py:__init__():39] meta init
+2021-07-09 12:20:22,116 DEBUG   HandlerThread:32095 [meta.py:__init__():53] meta init done
+2021-07-09 12:20:22,116 DEBUG   HandlerThread:32095 [meta.py:probe():210] probe
+2021-07-09 12:20:22,117 DEBUG   HandlerThread:32095 [meta.py:_setup_git():200] setup git
+2021-07-09 12:20:22,137 DEBUG   HandlerThread:32095 [meta.py:_setup_git():207] setup git done
+2021-07-09 12:20:22,138 DEBUG   HandlerThread:32095 [meta.py:_save_pip():57] save pip
+2021-07-09 12:20:22,138 DEBUG   HandlerThread:32095 [meta.py:_save_pip():71] save pip done
+2021-07-09 12:20:22,138 DEBUG   HandlerThread:32095 [meta.py:probe():252] probe done
+2021-07-09 12:20:22,170 DEBUG   SenderThread:32095 [sender.py:send():179] send: files
+2021-07-09 12:20:22,171 INFO    SenderThread:32095 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now
+2021-07-09 12:20:22,175 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 12:20:22,176 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 12:20:22,201 DEBUG   SenderThread:32095 [sender.py:send():179] send: config
+2021-07-09 12:20:22,202 DEBUG   SenderThread:32095 [sender.py:send():179] send: config
+2021-07-09 12:20:22,202 DEBUG   SenderThread:32095 [sender.py:send():179] send: config
+2021-07-09 12:20:22,407 INFO    Thread-11 :32095 [upload_job.py:push():137] Uploaded file /tmp/tmp756bbzv5wandb/1jtttinj-wandb-metadata.json
+2021-07-09 12:20:23,116 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/wandb-summary.json
+2021-07-09 12:20:23,116 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/wandb-metadata.json
+2021-07-09 12:20:23,116 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:20:23,116 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/requirements.txt
+2021-07-09 12:20:25,115 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:20:29,117 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:20:31,118 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:20:37,225 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 12:20:37,225 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 12:20:50,195 DEBUG   SenderThread:32095 [sender.py:send():179] send: stats
+2021-07-09 12:20:52,274 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 12:20:52,275 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 12:20:53,127 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/config.yaml
+2021-07-09 12:21:07,310 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 12:21:07,310 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 12:21:20,275 DEBUG   SenderThread:32095 [sender.py:send():179] send: stats
+2021-07-09 12:21:22,366 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 12:21:22,366 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 12:21:37,397 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 12:21:37,397 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 12:21:43,148 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:21:45,149 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:21:47,150 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:21:49,151 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:21:50,353 DEBUG   SenderThread:32095 [sender.py:send():179] send: stats
+2021-07-09 12:21:51,152 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:21:52,475 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 12:21:52,475 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 12:21:53,153 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:21:55,154 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:21:57,155 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:21:59,156 INFO    Thread-8  :32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:22:00,555 DEBUG   SenderThread:32095 [sender.py:send():179] send: telemetry
+2021-07-09 12:22:00,555 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 12:22:00,555 DEBUG   SenderThread:32095 [sender.py:send():179] send: exit
+2021-07-09 12:22:00,555 INFO    SenderThread:32095 [sender.py:send_exit():287] handling exit code: 1
+2021-07-09 12:22:00,556 INFO    SenderThread:32095 [sender.py:send_exit():295] send defer
+2021-07-09 12:22:00,556 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 12:22:00,556 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 12:22:00,556 INFO    HandlerThread:32095 [handler.py:handle_request_defer():141] handle defer: 0
+2021-07-09 12:22:00,557 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: defer
+2021-07-09 12:22:00,557 INFO    SenderThread:32095 [sender.py:send_request_defer():304] handle sender defer: 0
+2021-07-09 12:22:00,557 INFO    SenderThread:32095 [sender.py:transition_state():308] send defer: 1
+2021-07-09 12:22:00,557 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 12:22:00,557 INFO    HandlerThread:32095 [handler.py:handle_request_defer():141] handle defer: 1
+2021-07-09 12:22:00,578 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: defer
+2021-07-09 12:22:00,578 INFO    SenderThread:32095 [sender.py:send_request_defer():304] handle sender defer: 1
+2021-07-09 12:22:00,578 INFO    SenderThread:32095 [sender.py:transition_state():308] send defer: 2
+2021-07-09 12:22:00,579 DEBUG   SenderThread:32095 [sender.py:send():179] send: stats
+2021-07-09 12:22:00,579 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 12:22:00,579 INFO    HandlerThread:32095 [handler.py:handle_request_defer():141] handle defer: 2
+2021-07-09 12:22:00,579 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: defer
+2021-07-09 12:22:00,579 INFO    SenderThread:32095 [sender.py:send_request_defer():304] handle sender defer: 2
+2021-07-09 12:22:00,579 INFO    SenderThread:32095 [sender.py:transition_state():308] send defer: 3
+2021-07-09 12:22:00,579 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 12:22:00,580 INFO    HandlerThread:32095 [handler.py:handle_request_defer():141] handle defer: 3
+2021-07-09 12:22:00,580 DEBUG   SenderThread:32095 [sender.py:send():179] send: summary
+2021-07-09 12:22:00,580 INFO    SenderThread:32095 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-09 12:22:00,580 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: defer
+2021-07-09 12:22:00,580 INFO    SenderThread:32095 [sender.py:send_request_defer():304] handle sender defer: 3
+2021-07-09 12:22:00,580 INFO    SenderThread:32095 [sender.py:transition_state():308] send defer: 4
+2021-07-09 12:22:00,580 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 12:22:00,580 INFO    HandlerThread:32095 [handler.py:handle_request_defer():141] handle defer: 4
+2021-07-09 12:22:00,581 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: defer
+2021-07-09 12:22:00,581 INFO    SenderThread:32095 [sender.py:send_request_defer():304] handle sender defer: 4
+2021-07-09 12:22:00,651 INFO    SenderThread:32095 [sender.py:transition_state():308] send defer: 5
+2021-07-09 12:22:00,652 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 12:22:00,652 INFO    HandlerThread:32095 [handler.py:handle_request_defer():141] handle defer: 5
+2021-07-09 12:22:00,652 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: defer
+2021-07-09 12:22:00,652 INFO    SenderThread:32095 [sender.py:send_request_defer():304] handle sender defer: 5
+2021-07-09 12:22:00,652 INFO    SenderThread:32095 [dir_watcher.py:finish():282] shutting down directory watcher
+2021-07-09 12:22:00,658 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 12:22:01,157 INFO    SenderThread:32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/wandb-summary.json
+2021-07-09 12:22:01,158 INFO    SenderThread:32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:22:01,158 INFO    SenderThread:32095 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/config.yaml
+2021-07-09 12:22:01,158 INFO    SenderThread:32095 [dir_watcher.py:finish():312] scan: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files
+2021-07-09 12:22:01,158 INFO    SenderThread:32095 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/requirements.txt requirements.txt
+2021-07-09 12:22:01,158 INFO    SenderThread:32095 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log output.log
+2021-07-09 12:22:01,158 INFO    SenderThread:32095 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/wandb-metadata.json wandb-metadata.json
+2021-07-09 12:22:01,158 INFO    SenderThread:32095 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/config.yaml config.yaml
+2021-07-09 12:22:01,158 INFO    SenderThread:32095 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/wandb-summary.json wandb-summary.json
+2021-07-09 12:22:01,159 INFO    SenderThread:32095 [sender.py:transition_state():308] send defer: 6
+2021-07-09 12:22:01,159 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 12:22:01,162 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 12:22:01,162 INFO    HandlerThread:32095 [handler.py:handle_request_defer():141] handle defer: 6
+2021-07-09 12:22:01,165 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: defer
+2021-07-09 12:22:01,165 INFO    SenderThread:32095 [sender.py:send_request_defer():304] handle sender defer: 6
+2021-07-09 12:22:01,165 INFO    SenderThread:32095 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-09 12:22:01,263 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 12:22:01,264 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 12:22:01,365 INFO    Thread-14 :32095 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/config.yaml
+2021-07-09 12:22:01,365 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 12:22:01,366 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 12:22:01,372 INFO    Thread-13 :32095 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/output.log
+2021-07-09 12:22:01,388 INFO    Thread-15 :32095 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/wandb-summary.json
+2021-07-09 12:22:01,392 INFO    Thread-12 :32095 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/files/requirements.txt
+2021-07-09 12:22:01,467 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 12:22:01,468 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 12:22:01,569 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 12:22:01,570 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 12:22:01,592 INFO    Thread-7  :32095 [sender.py:transition_state():308] send defer: 7
+2021-07-09 12:22:01,593 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 12:22:01,593 INFO    HandlerThread:32095 [handler.py:handle_request_defer():141] handle defer: 7
+2021-07-09 12:22:01,593 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: defer
+2021-07-09 12:22:01,593 INFO    SenderThread:32095 [sender.py:send_request_defer():304] handle sender defer: 7
+2021-07-09 12:22:01,663 INFO    SenderThread:32095 [sender.py:transition_state():308] send defer: 8
+2021-07-09 12:22:01,664 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 12:22:01,664 INFO    HandlerThread:32095 [handler.py:handle_request_defer():141] handle defer: 8
+2021-07-09 12:22:01,664 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: defer
+2021-07-09 12:22:01,664 INFO    SenderThread:32095 [sender.py:send_request_defer():304] handle sender defer: 8
+2021-07-09 12:22:01,664 INFO    SenderThread:32095 [sender.py:transition_state():308] send defer: 9
+2021-07-09 12:22:01,665 DEBUG   SenderThread:32095 [sender.py:send():179] send: final
+2021-07-09 12:22:01,665 DEBUG   SenderThread:32095 [sender.py:send():179] send: footer
+2021-07-09 12:22:01,665 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 12:22:01,665 INFO    HandlerThread:32095 [handler.py:handle_request_defer():141] handle defer: 9
+2021-07-09 12:22:01,665 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: defer
+2021-07-09 12:22:01,665 INFO    SenderThread:32095 [sender.py:send_request_defer():304] handle sender defer: 9
+2021-07-09 12:22:01,671 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 12:22:01,671 DEBUG   SenderThread:32095 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 12:22:01,672 INFO    SenderThread:32095 [file_pusher.py:join():182] waiting for file pusher
+2021-07-09 12:22:01,673 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: get_summary
+2021-07-09 12:22:01,674 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: sampled_history
+2021-07-09 12:22:01,674 DEBUG   HandlerThread:32095 [handler.py:handle_request():124] handle_request: shutdown
+2021-07-09 12:22:01,674 INFO    HandlerThread:32095 [handler.py:finish():638] shutting down handler
+2021-07-09 12:22:02,665 INFO    WriterThread:32095 [datastore.py:close():288] close: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/run-2porvgvv.wandb
+2021-07-09 12:22:02,672 INFO    SenderThread:32095 [sender.py:finish():945] shutting down sender
+2021-07-09 12:22:02,672 INFO    SenderThread:32095 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-09 12:22:02,672 INFO    SenderThread:32095 [file_pusher.py:join():182] waiting for file pusher
+2021-07-09 12:22:02,675 INFO    MainThread:32095 [internal.py:handle_exit():78] Internal process exited

wandb/run-20210709_122021-2porvgvv/logs/debug.log

@@ -0,0 +1,88 @@
+2021-07-09 12:20:21,344 INFO    MainThread:31038 [wandb_setup.py:_flush():69] Unhandled environment var: WANDB_WATCH
+2021-07-09 12:20:21,344 INFO    MainThread:31038 [wandb_setup.py:_flush():69] setting env: {'project': 'hf-flax-robert-base-mr', 'entity': 'nipunsadvilkar'}
+2021-07-09 12:20:21,344 INFO    MainThread:31038 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-07-09 12:20:21,344 INFO    MainThread:31038 [wandb_init.py:_log_setup():337] Logging user logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/logs/debug.log
+2021-07-09 12:20:21,344 INFO    MainThread:31038 [wandb_init.py:_log_setup():338] Logging internal logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122021-2porvgvv/logs/debug-internal.log
+2021-07-09 12:20:21,344 INFO    MainThread:31038 [wandb_init.py:init():370] calling init triggers
+2021-07-09 12:20:21,344 INFO    MainThread:31038 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
+config: {}
+2021-07-09 12:20:21,344 INFO    MainThread:31038 [wandb_init.py:init():419] starting backend
+2021-07-09 12:20:21,344 INFO    MainThread:31038 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-07-09 12:20:21,378 INFO    MainThread:31038 [backend.py:ensure_launched():135] starting backend process...
+2021-07-09 12:20:21,411 INFO    MainThread:31038 [backend.py:ensure_launched():139] started backend process with pid: 32095
+2021-07-09 12:20:21,413 INFO    MainThread:31038 [wandb_init.py:init():424] backend started and connected
+2021-07-09 12:20:21,416 INFO    MainThread:31038 [wandb_init.py:init():472] updated telemetry
+2021-07-09 12:20:21,417 INFO    MainThread:31038 [wandb_init.py:init():491] communicating current version
+2021-07-09 12:20:22,044 INFO    MainThread:31038 [wandb_init.py:init():496] got version response 
+2021-07-09 12:20:22,044 INFO    MainThread:31038 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
+2021-07-09 12:20:22,114 INFO    MainThread:31038 [wandb_init.py:init():529] starting run threads in backend
+2021-07-09 12:20:22,173 INFO    MainThread:31038 [wandb_run.py:_console_start():1623] atexit reg
+2021-07-09 12:20:22,174 INFO    MainThread:31038 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
+2021-07-09 12:20:22,174 INFO    MainThread:31038 [wandb_run.py:_redirect():1502] Redirecting console.
+2021-07-09 12:20:22,176 INFO    MainThread:31038 [wandb_run.py:_redirect():1558] Redirects installed.
+2021-07-09 12:20:22,177 INFO    MainThread:31038 [wandb_init.py:init():554] run started, returning control to user process
+2021-07-09 12:20:22,178 INFO    MainThread:31038 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 4, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 8.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul09_12-20-15_t1v-n-112df4a9-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 500, 'save_total_limit': None, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
+2021-07-09 12:20:22,179 INFO    MainThread:31038 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'roberta', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
+2021-07-09 12:20:22,179 INFO    MainThread:31038 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': 'oscar', 'dataset_config_name': 'unshuffled_deduplicated_als', 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 128, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False}
+2021-07-09 12:21:58,359 INFO    MainThread:31038 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1
+2021-07-09 12:21:58,359 INFO    MainThread:31038 [wandb_run.py:_restore():1565] restore
+2021-07-09 12:22:00,556 INFO    MainThread:31038 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 1200
+  total_bytes: 1200
+}
+
+2021-07-09 12:22:01,162 INFO    MainThread:31038 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 4
+}
+pusher_stats {
+  uploaded_bytes: 1200
+  total_bytes: 9461
+}
+
+2021-07-09 12:22:01,264 INFO    MainThread:31038 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 9463
+  total_bytes: 9463
+}
+
+2021-07-09 12:22:01,366 INFO    MainThread:31038 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 9463
+  total_bytes: 9463
+}
+
+2021-07-09 12:22:01,469 INFO    MainThread:31038 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 9463
+  total_bytes: 9463
+}
+
+2021-07-09 12:22:01,570 INFO    MainThread:31038 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 9463
+  total_bytes: 9463
+}
+
+2021-07-09 12:22:01,672 INFO    MainThread:31038 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true
+exit_result {
+}
+file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 9463
+  total_bytes: 9463
+}
+
+2021-07-09 12:22:02,789 INFO    MainThread:31038 [wandb_run.py:_show_files():1937] logging synced files

wandb/run-20210709_122709-2pc90wcd/files/config.yaml

@@ -0,0 +1,303 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.10.33
+    framework: huggingface
+    huggingface_version: 4.9.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.8.10
+    t:
+      1:
+      - 11
+      4: 3.8.10
+      5: 0.10.33
+      6: 4.9.0.dev0
+      8:
+      - 5
+adafactor:
+  desc: null
+  value: false
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+cache_dir:
+  desc: null
+  value: null
+config_name:
+  desc: null
+  value: ./
+dataloader_drop_last:
+  desc: null
+  value: false
+dataloader_num_workers:
+  desc: null
+  value: 0
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataset_config_name:
+  desc: null
+  value: unshuffled_deduplicated_als
+dataset_name:
+  desc: null
+  value: oscar
+ddp_find_unused_parameters:
+  desc: null
+  value: null
+debug:
+  desc: null
+  value: []
+deepspeed:
+  desc: null
+  value: null
+disable_tqdm:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: false
+do_predict:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+dtype:
+  desc: null
+  value: float32
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_steps:
+  desc: null
+  value: 500
+evaluation_strategy:
+  desc: null
+  value: IntervalStrategy.NO
+fp16:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+fp16_full_eval:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+gradient_accumulation_steps:
+  desc: null
+  value: 1
+greater_is_better:
+  desc: null
+  value: null
+group_by_length:
+  desc: null
+  value: false
+ignore_data_skip:
+  desc: null
+  value: false
+label_names:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+learning_rate:
+  desc: null
+  value: 0.0003
+length_column_name:
+  desc: null
+  value: length
+line_by_line:
+  desc: null
+  value: false
+load_best_model_at_end:
+  desc: null
+  value: false
+local_rank:
+  desc: null
+  value: -1
+log_level:
+  desc: null
+  value: -1
+log_level_replica:
+  desc: null
+  value: -1
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: ./runs/Jul09_12-27-03_t1v-n-112df4a9-w-0
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 500
+logging_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+lr_scheduler_type:
+  desc: null
+  value: SchedulerType.LINEAR
+max_grad_norm:
+  desc: null
+  value: 1.0
+max_seq_length:
+  desc: null
+  value: 128
+max_steps:
+  desc: null
+  value: -1
+metric_for_best_model:
+  desc: null
+  value: null
+mlm_probability:
+  desc: null
+  value: 0.15
+model_name_or_path:
+  desc: null
+  value: null
+model_type:
+  desc: null
+  value: roberta
+mp_parameters:
+  desc: null
+  value: ''
+no_cuda:
+  desc: null
+  value: false
+num_train_epochs:
+  desc: null
+  value: 8.0
+output_dir:
+  desc: null
+  value: ./
+overwrite_cache:
+  desc: null
+  value: false
+overwrite_output_dir:
+  desc: null
+  value: true
+pad_to_max_length:
+  desc: null
+  value: false
+past_index:
+  desc: null
+  value: -1
+per_device_eval_batch_size:
+  desc: null
+  value: 4
+per_device_train_batch_size:
+  desc: null
+  value: 4
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+prediction_loss_only:
+  desc: null
+  value: false
+preprocessing_num_workers:
+  desc: null
+  value: null
+push_to_hub:
+  desc: null
+  value: false
+push_to_hub_model_id:
+  desc: null
+  value: ''
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: null
+remove_unused_columns:
+  desc: null
+  value: true
+report_to:
+  desc: null
+  value:
+  - tensorboard
+  - wandb
+resume_from_checkpoint:
+  desc: null
+  value: null
+run_name:
+  desc: null
+  value: ./
+save_on_each_node:
+  desc: null
+  value: false
+save_steps:
+  desc: null
+  value: 500
+save_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+save_total_limit:
+  desc: null
+  value: null
+seed:
+  desc: null
+  value: 42
+sharded_ddp:
+  desc: null
+  value: []
+skip_memory_metrics:
+  desc: null
+  value: true
+tokenizer_name:
+  desc: null
+  value: ./
+tpu_metrics_debug:
+  desc: null
+  value: false
+tpu_num_cores:
+  desc: null
+  value: null
+train_file:
+  desc: null
+  value: null
+train_ref_file:
+  desc: null
+  value: null
+use_fast_tokenizer:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+validation_file:
+  desc: null
+  value: null
+validation_ref_file:
+  desc: null
+  value: null
+validation_split_percentage:
+  desc: null
+  value: 5
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 1000
+weight_decay:
+  desc: null
+  value: 0.0

wandb/run-20210709_122709-2pc90wcd/files/output.log

@@ -0,0 +1,18 @@
+/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
+  warnings.warn(
+/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
+  warnings.warn(
+Epoch ... (1/8):   0%|                                                                                                                                                                | 0/8 [00:00<?, ?it/s]
+
+
+Epoch ... (1/8):  12%|███████████████████                                                                                                                                     | 1/8 [01:15<08:49, 75.59s/it]
+
+
+Epoch ... (1/8):  25%|██████████████████████████████████████                                                                                                                  | 2/8 [01:20<03:23, 33.95s/it]
+
+Epoch ... (1/8):  38%|█████████████████████████████████████████████████████████                                                                                               | 3/8 [01:25<01:43, 20.77s/it]
+
+Training...:  51%|█████████████████████████████████████████████████████████████████████████████▌                                                                           | 72/142 [00:02<00:01, 35.13it/s]
+Step... (500 | Loss: 8.018753051757812, Learning Rate: 0.0001500000071246177)
+
+Step... (500 | Loss: 8.205772399902344, Acc: 0.0773010179400444):  38%|██████████████████████████████████████▋                                                                | 3/8 [01:33<02:35, 31.10s/it]

wandb/run-20210709_122709-2pc90wcd/files/requirements.txt

@@ -0,0 +1,96 @@
+absl-py==0.13.0
+aiohttp==3.7.4.post0
+async-timeout==3.0.1
+attrs==21.2.0
+backcall==0.2.0
+cachetools==4.2.2
+certifi==2021.5.30
+chardet==4.0.0
+chex==0.0.8
+click==8.0.1
+configparser==5.0.2
+cycler==0.10.0
+datasets==1.9.1.dev0
+decorator==5.0.9
+dill==0.3.4
+dm-tree==0.1.6
+docker-pycreds==0.4.0
+filelock==3.0.12
+flatbuffers==2.0
+flax==0.3.4
+fsspec==2021.6.1
+gitdb==4.0.7
+gitpython==3.1.18
+google-auth-oauthlib==0.4.4
+google-auth==1.32.1
+grpcio==1.38.1
+huggingface-hub==0.0.12
+idna==2.10
+ipython-genutils==0.2.0
+ipython==7.25.0
+jax==0.2.16
+jaxlib==0.1.68
+jedi==0.18.0
+joblib==1.0.1
+kiwisolver==1.3.1
+libtpu-nightly==0.1.dev20210615
+markdown==3.3.4
+matplotlib-inline==0.1.2
+matplotlib==3.4.2
+msgpack==1.0.2
+multidict==5.1.0
+multiprocess==0.70.12.2
+numpy==1.21.0
+oauthlib==3.1.1
+opt-einsum==3.3.0
+optax==0.0.9
+packaging==21.0
+pandas==1.3.0
+parso==0.8.2
+pathtools==0.1.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==8.3.1
+pip==20.0.2
+pkg-resources==0.0.0
+promise==2.3
+prompt-toolkit==3.0.19
+protobuf==3.17.3
+psutil==5.8.0
+ptyprocess==0.7.0
+pyarrow==4.0.1
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pygments==2.9.0
+pyparsing==2.4.7
+python-dateutil==2.8.1
+pytz==2021.1
+pyyaml==5.4.1
+regex==2021.7.6
+requests-oauthlib==1.3.0
+requests==2.25.1
+rsa==4.7.2
+sacremoses==0.0.45
+scipy==1.7.0
+sentry-sdk==1.3.0
+setuptools==44.0.0
+shortuuid==1.0.1
+six==1.16.0
+smmap==4.0.0
+subprocess32==3.5.4
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.5.0
+tokenizers==0.10.3
+toolz==0.11.1
+tqdm==4.61.2
+traitlets==5.0.5
+transformers==4.9.0.dev0
+typing-extensions==3.10.0.0
+urllib3==1.26.6
+wandb==0.10.33
+wcwidth==0.2.5
+werkzeug==2.0.1
+wheel==0.36.2
+xxhash==2.0.2
+yarl==1.6.3

wandb/run-20210709_122709-2pc90wcd/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+    "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
+    "python": "3.8.10",
+    "heartbeatAt": "2021-07-09T12:27:09.785328",
+    "startedAt": "2021-07-09T12:27:09.051375",
+    "docker": null,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [
+        "--output_dir=./",
+        "--model_type=roberta",
+        "--config_name=./",
+        "--tokenizer_name=./",
+        "--dataset_name=oscar",
+        "--dataset_config_name=unshuffled_deduplicated_als",
+        "--max_seq_length=128",
+        "--per_device_train_batch_size=4",
+        "--per_device_eval_batch_size=4",
+        "--learning_rate=3e-4",
+        "--warmup_steps=1000",
+        "--overwrite_output_dir",
+        "--num_train_epochs=8"
+    ],
+    "state": "running",
+    "program": "run_mlm_flax.py",
+    "codePath": "run_mlm_flax.py",
+    "git": {
+        "remote": "https://huggingface.co/flax-community/roberta-base-mr",
+        "commit": "faa64e8a79301832a56c78b2b293675039070bb0"
+    },
+    "email": "nipunsadvilkar@gmail.com",
+    "root": "/home/nipunsadvilkar/sample/roberta-base-mr",
+    "host": "t1v-n-112df4a9-w-0",
+    "username": "nipunsadvilkar",
+    "executable": "/home/nipunsadvilkar/roberta_mr_env/bin/python"
+}

wandb/run-20210709_122709-2pc90wcd/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train_time": 3.0330424308776855, "_runtime": 99, "_timestamp": 1625833728, "_step": 303, "training_learning_rate": {"_type": "histogram", "values": [3, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 3], "bins": [0.00012780001270584762, 0.00012849376071244478, 0.00012918750871904194, 0.0001298812567256391, 0.00013057500473223627, 0.00013126876729074866, 0.00013196251529734582, 0.00013265626330394298, 0.00013335001131054014, 0.0001340437593171373, 0.00013473750732373446, 0.00013543125533033162, 0.000136125017888844, 0.00013681876589544117, 0.00013751251390203834, 0.0001382062619086355, 0.00013890000991523266, 0.00013959375792182982, 0.00014028750592842698, 0.00014098125393502414, 0.0001416750019416213, 0.0001423687645001337, 0.00014306251250673085, 0.00014375626051332802, 0.00014445000851992518, 0.00014514375652652234, 0.0001458375045331195, 0.00014653125253971666, 0.00014722501509822905, 0.0001479187631048262, 0.00014861251111142337, 0.00014930625911802053, 0.0001500000071246177]}, "train_step": 500, "train_learning_rate": 0.0001500000071246177, "training_loss": {"_type": "histogram", "values": [1, 0, 1, 2, 2, 1, 2, 2, 2, 6, 3, 2, 2, 2, 6, 2, 3, 3, 0, 3, 5, 3, 3, 2, 2, 3, 1, 1, 1, 2, 5, 2], "bins": [7.636768341064453, 7.65283203125, 7.668895721435547, 7.684959411621094, 7.701023101806641, 7.7170867919921875, 7.733150482177734, 7.749214172363281, 7.765277862548828, 7.781341552734375, 7.797405242919922, 7.813468933105469, 7.829532623291016, 7.8455963134765625, 7.861660003662109, 7.877723693847656, 7.893787384033203, 7.90985107421875, 7.925914764404297, 7.941978454589844, 7.958042144775391, 7.9741058349609375, 7.990169525146484, 8.006233215332031, 8.022296905517578, 8.038360595703125, 8.054424285888672, 8.070487976074219, 8.086551666259766, 8.102615356445312, 8.11867904663086, 8.134742736816406, 8.150806427001953]}, "train_loss": 8.018753051757812, "eval_step": 426}

wandb/run-20210709_122709-2pc90wcd/logs/debug.log

@@ -0,0 +1,53 @@
+2021-07-09 12:27:09,053 INFO    MainThread:32339 [wandb_setup.py:_flush():69] Unhandled environment var: WANDB_WATCH
+2021-07-09 12:27:09,053 INFO    MainThread:32339 [wandb_setup.py:_flush():69] setting env: {'project': 'hf-flax-robert-base-mr', 'entity': 'nipunsadvilkar'}
+2021-07-09 12:27:09,053 INFO    MainThread:32339 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-07-09 12:27:09,053 INFO    MainThread:32339 [wandb_init.py:_log_setup():337] Logging user logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/logs/debug.log
+2021-07-09 12:27:09,053 INFO    MainThread:32339 [wandb_init.py:_log_setup():338] Logging internal logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/logs/debug-internal.log
+2021-07-09 12:27:09,053 INFO    MainThread:32339 [wandb_init.py:init():370] calling init triggers
+2021-07-09 12:27:09,053 INFO    MainThread:32339 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
+config: {}
+2021-07-09 12:27:09,053 INFO    MainThread:32339 [wandb_init.py:init():419] starting backend
+2021-07-09 12:27:09,053 INFO    MainThread:32339 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-07-09 12:27:09,087 INFO    MainThread:32339 [backend.py:ensure_launched():135] starting backend process...
+2021-07-09 12:27:09,119 INFO    MainThread:32339 [backend.py:ensure_launched():139] started backend process with pid: 33395
+2021-07-09 12:27:09,122 INFO    MainThread:32339 [wandb_init.py:init():424] backend started and connected
+2021-07-09 12:27:09,124 INFO    MainThread:32339 [wandb_init.py:init():472] updated telemetry
+2021-07-09 12:27:09,125 INFO    MainThread:32339 [wandb_init.py:init():491] communicating current version
+2021-07-09 12:27:09,709 INFO    MainThread:32339 [wandb_init.py:init():496] got version response 
+2021-07-09 12:27:09,709 INFO    MainThread:32339 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
+2021-07-09 12:27:09,783 INFO    MainThread:32339 [wandb_init.py:init():529] starting run threads in backend
+2021-07-09 12:27:09,838 INFO    MainThread:32339 [wandb_run.py:_console_start():1623] atexit reg
+2021-07-09 12:27:09,839 INFO    MainThread:32339 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
+2021-07-09 12:27:09,839 INFO    MainThread:32339 [wandb_run.py:_redirect():1502] Redirecting console.
+2021-07-09 12:27:09,841 INFO    MainThread:32339 [wandb_run.py:_redirect():1558] Redirects installed.
+2021-07-09 12:27:09,841 INFO    MainThread:32339 [wandb_init.py:init():554] run started, returning control to user process
+2021-07-09 12:27:09,842 INFO    MainThread:32339 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 4, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 8.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul09_12-27-03_t1v-n-112df4a9-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 500, 'save_total_limit': None, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
+2021-07-09 12:27:09,843 INFO    MainThread:32339 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'roberta', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
+2021-07-09 12:27:09,844 INFO    MainThread:32339 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': 'oscar', 'dataset_config_name': 'unshuffled_deduplicated_als', 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 128, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False}
+2021-07-09 12:28:48,239 INFO    MainThread:32339 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1
+2021-07-09 12:28:48,240 INFO    MainThread:32339 [wandb_run.py:_restore():1565] restore
+2021-07-09 12:28:52,518 ERROR   MainThread:32339 [wandb_run.py:_atexit_cleanup():1611] Problem finishing run
+Traceback (most recent call last):
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/wandb_run.py", line 1601, in _atexit_cleanup
+    self._on_finish()
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/wandb_run.py", line 1737, in _on_finish
+    self.history._flush()
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/wandb_history.py", line 59, in _flush
+    self._callback(row=self._data, step=self._step)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/wandb_run.py", line 931, in _history_callback
+    self._backend.interface.publish_history(
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 231, in publish_history
+    item.value_json = json_dumps_safer_history(v)  # type: ignore
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/util.py", line 667, in json_dumps_safer_history
+    return json.dumps(obj, cls=WandBHistoryJSONEncoder, **kwargs)
+  File "/usr/lib/python3.8/json/__init__.py", line 234, in dumps
+    return cls(
+  File "/usr/lib/python3.8/json/encoder.py", line 199, in encode
+    chunks = self.iterencode(o, _one_shot=True)
+  File "/usr/lib/python3.8/json/encoder.py", line 257, in iterencode
+    return _iterencode(o, 0)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/util.py", line 634, in default
+    return json.JSONEncoder.default(self, obj)
+  File "/usr/lib/python3.8/json/encoder.py", line 179, in default
+    raise TypeError(f'Object of type {o.__class__.__name__} '
+TypeError: Object of type DeviceArray is not JSON serializable

wandb/run-20210709_124353-3pjvcule/files/config.yaml

@@ -0,0 +1,19 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.10.33
+    framework: huggingface
+    huggingface_version: 4.9.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.8.10
+    t:
+      1:
+      - 11
+      4: 3.8.10
+      5: 0.10.33
+      6: 4.9.0.dev0
+      8:
+      - 5

wandb/run-20210709_124353-3pjvcule/files/output.log

@@ -0,0 +1,5 @@
+/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
+  warnings.warn(
+/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
+  warnings.warn(
+Epoch ... (1/8):   0%|                                                                                                                                                                | 0/8 [00:00<?, ?it/s]

wandb/run-20210709_124353-3pjvcule/files/requirements.txt

@@ -0,0 +1,96 @@
+absl-py==0.13.0
+aiohttp==3.7.4.post0
+async-timeout==3.0.1
+attrs==21.2.0
+backcall==0.2.0
+cachetools==4.2.2
+certifi==2021.5.30
+chardet==4.0.0
+chex==0.0.8
+click==8.0.1
+configparser==5.0.2
+cycler==0.10.0
+datasets==1.9.1.dev0
+decorator==5.0.9
+dill==0.3.4
+dm-tree==0.1.6
+docker-pycreds==0.4.0
+filelock==3.0.12
+flatbuffers==2.0
+flax==0.3.4
+fsspec==2021.6.1
+gitdb==4.0.7
+gitpython==3.1.18
+google-auth-oauthlib==0.4.4
+google-auth==1.32.1
+grpcio==1.38.1
+huggingface-hub==0.0.12
+idna==2.10
+ipython-genutils==0.2.0
+ipython==7.25.0
+jax==0.2.16
+jaxlib==0.1.68
+jedi==0.18.0
+joblib==1.0.1
+kiwisolver==1.3.1
+libtpu-nightly==0.1.dev20210615
+markdown==3.3.4
+matplotlib-inline==0.1.2
+matplotlib==3.4.2
+msgpack==1.0.2
+multidict==5.1.0
+multiprocess==0.70.12.2
+numpy==1.21.0
+oauthlib==3.1.1
+opt-einsum==3.3.0
+optax==0.0.9
+packaging==21.0
+pandas==1.3.0
+parso==0.8.2
+pathtools==0.1.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==8.3.1
+pip==20.0.2
+pkg-resources==0.0.0
+promise==2.3
+prompt-toolkit==3.0.19
+protobuf==3.17.3
+psutil==5.8.0
+ptyprocess==0.7.0
+pyarrow==4.0.1
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pygments==2.9.0
+pyparsing==2.4.7
+python-dateutil==2.8.1
+pytz==2021.1
+pyyaml==5.4.1
+regex==2021.7.6
+requests-oauthlib==1.3.0
+requests==2.25.1
+rsa==4.7.2
+sacremoses==0.0.45
+scipy==1.7.0
+sentry-sdk==1.3.0
+setuptools==44.0.0
+shortuuid==1.0.1
+six==1.16.0
+smmap==4.0.0
+subprocess32==3.5.4
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.5.0
+tokenizers==0.10.3
+toolz==0.11.1
+tqdm==4.61.2
+traitlets==5.0.5
+transformers==4.9.0.dev0
+typing-extensions==3.10.0.0
+urllib3==1.26.6
+wandb==0.10.33
+wcwidth==0.2.5
+werkzeug==2.0.1
+wheel==0.36.2
+xxhash==2.0.2
+yarl==1.6.3

wandb/run-20210709_124353-3pjvcule/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+    "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
+    "python": "3.8.10",
+    "heartbeatAt": "2021-07-09T12:43:54.473948",
+    "startedAt": "2021-07-09T12:43:53.659249",
+    "docker": null,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [
+        "--output_dir=./",
+        "--model_type=roberta",
+        "--config_name=./",
+        "--tokenizer_name=./",
+        "--dataset_name=oscar",
+        "--dataset_config_name=unshuffled_deduplicated_als",
+        "--max_seq_length=128",
+        "--per_device_train_batch_size=4",
+        "--per_device_eval_batch_size=4",
+        "--learning_rate=3e-4",
+        "--warmup_steps=1000",
+        "--overwrite_output_dir",
+        "--num_train_epochs=8"
+    ],
+    "state": "running",
+    "program": "run_mlm_flax.py",
+    "codePath": "run_mlm_flax.py",
+    "git": {
+        "remote": "https://huggingface.co/flax-community/roberta-base-mr",
+        "commit": "faa64e8a79301832a56c78b2b293675039070bb0"
+    },
+    "email": "nipunsadvilkar@gmail.com",
+    "root": "/home/nipunsadvilkar/sample/roberta-base-mr",
+    "host": "t1v-n-112df4a9-w-0",
+    "username": "nipunsadvilkar",
+    "executable": "/home/nipunsadvilkar/roberta_mr_env/bin/python"
+}

wandb/run-20210709_124353-3pjvcule/files/wandb-summary.json

@@ -0,0 +1 @@
+{}

wandb/run-20210709_124353-3pjvcule/logs/debug-internal.log

@@ -0,0 +1,55 @@
+2021-07-09 12:43:54,336 INFO    MainThread:34715 [internal.py:wandb_internal():88] W&B internal server running at pid: 34715, started at: 2021-07-09 12:43:54.336084
+2021-07-09 12:43:54,338 DEBUG   HandlerThread:34715 [handler.py:handle_request():124] handle_request: check_version
+2021-07-09 12:43:54,339 INFO    WriterThread:34715 [datastore.py:open_for_write():80] open: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/run-3pjvcule.wandb
+2021-07-09 12:43:54,340 DEBUG   SenderThread:34715 [sender.py:send():179] send: header
+2021-07-09 12:43:54,340 DEBUG   SenderThread:34715 [sender.py:send_request():193] send_request: check_version
+2021-07-09 12:43:54,404 DEBUG   SenderThread:34715 [sender.py:send():179] send: run
+2021-07-09 12:43:54,471 INFO    SenderThread:34715 [dir_watcher.py:__init__():168] watching files in: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files
+2021-07-09 12:43:54,471 INFO    SenderThread:34715 [sender.py:_start_run_threads():716] run started: 3pjvcule with start time 1625834633
+2021-07-09 12:43:54,471 DEBUG   SenderThread:34715 [sender.py:send():179] send: summary
+2021-07-09 12:43:54,472 INFO    SenderThread:34715 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-09 12:43:54,472 DEBUG   HandlerThread:34715 [handler.py:handle_request():124] handle_request: run_start
+2021-07-09 12:43:54,473 DEBUG   HandlerThread:34715 [meta.py:__init__():39] meta init
+2021-07-09 12:43:54,473 DEBUG   HandlerThread:34715 [meta.py:__init__():53] meta init done
+2021-07-09 12:43:54,473 DEBUG   HandlerThread:34715 [meta.py:probe():210] probe
+2021-07-09 12:43:54,474 DEBUG   HandlerThread:34715 [meta.py:_setup_git():200] setup git
+2021-07-09 12:43:54,494 DEBUG   HandlerThread:34715 [meta.py:_setup_git():207] setup git done
+2021-07-09 12:43:54,494 DEBUG   HandlerThread:34715 [meta.py:_save_pip():57] save pip
+2021-07-09 12:43:54,494 DEBUG   HandlerThread:34715 [meta.py:_save_pip():71] save pip done
+2021-07-09 12:43:54,495 DEBUG   HandlerThread:34715 [meta.py:probe():252] probe done
+2021-07-09 12:43:54,526 DEBUG   SenderThread:34715 [sender.py:send():179] send: files
+2021-07-09 12:43:54,526 INFO    SenderThread:34715 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now
+2021-07-09 12:43:54,531 DEBUG   HandlerThread:34715 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 12:43:54,532 DEBUG   SenderThread:34715 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 12:43:54,558 DEBUG   SenderThread:34715 [sender.py:send():179] send: config
+2021-07-09 12:43:54,559 DEBUG   SenderThread:34715 [sender.py:send():179] send: config
+2021-07-09 12:43:54,559 DEBUG   SenderThread:34715 [sender.py:send():179] send: config
+2021-07-09 12:43:54,734 INFO    Thread-11 :34715 [upload_job.py:push():137] Uploaded file /tmp/tmp3gqpckyvwandb/1ct7grlp-wandb-metadata.json
+2021-07-09 12:43:55,472 INFO    Thread-8  :34715 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/requirements.txt
+2021-07-09 12:43:55,473 INFO    Thread-8  :34715 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/wandb-summary.json
+2021-07-09 12:43:55,473 INFO    Thread-8  :34715 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/wandb-metadata.json
+2021-07-09 12:43:55,473 INFO    Thread-8  :34715 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/output.log
+2021-07-09 12:44:01,474 INFO    Thread-8  :34715 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/output.log
+2021-07-09 12:44:03,475 INFO    Thread-8  :34715 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/output.log
+2021-07-09 12:44:09,435 WARNING MainThread:34715 [internal.py:wandb_internal():147] Internal process interrupt: 1
+2021-07-09 12:44:09,561 DEBUG   HandlerThread:34715 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 12:44:09,561 DEBUG   SenderThread:34715 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 12:44:10,614 WARNING MainThread:34715 [internal.py:wandb_internal():147] Internal process interrupt: 2
+2021-07-09 12:44:10,614 ERROR   MainThread:34715 [internal.py:wandb_internal():150] Internal process interrupted.
+2021-07-09 12:44:10,625 INFO    SenderThread:34715 [sender.py:finish():945] shutting down sender
+2021-07-09 12:44:10,626 INFO    SenderThread:34715 [dir_watcher.py:finish():282] shutting down directory watcher
+2021-07-09 12:44:10,626 INFO    HandlerThread:34715 [handler.py:finish():638] shutting down handler
+2021-07-09 12:44:11,514 INFO    SenderThread:34715 [dir_watcher.py:finish():312] scan: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files
+2021-07-09 12:44:11,514 INFO    SenderThread:34715 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/requirements.txt requirements.txt
+2021-07-09 12:44:11,514 INFO    SenderThread:34715 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/output.log output.log
+2021-07-09 12:44:11,515 INFO    SenderThread:34715 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/wandb-metadata.json wandb-metadata.json
+2021-07-09 12:44:11,515 INFO    SenderThread:34715 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/config.yaml config.yaml
+2021-07-09 12:44:11,515 INFO    SenderThread:34715 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/wandb-summary.json wandb-summary.json
+2021-07-09 12:44:11,515 INFO    SenderThread:34715 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-09 12:44:11,515 INFO    SenderThread:34715 [file_pusher.py:join():182] waiting for file pusher
+2021-07-09 12:44:11,570 INFO    WriterThread:34715 [datastore.py:close():288] close: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/run-3pjvcule.wandb
+2021-07-09 12:44:11,810 INFO    Thread-12 :34715 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/requirements.txt
+2021-07-09 12:44:11,811 INFO    Thread-14 :34715 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/config.yaml
+2021-07-09 12:44:11,824 INFO    Thread-13 :34715 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/output.log
+2021-07-09 12:44:11,856 INFO    Thread-15 :34715 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/files/wandb-summary.json
+2021-07-09 12:44:12,589 INFO    MainThread:34715 [internal.py:handle_exit():78] Internal process exited

wandb/run-20210709_124353-3pjvcule/logs/debug.log

@@ -0,0 +1,28 @@
+2021-07-09 12:43:53,661 INFO    MainThread:33657 [wandb_setup.py:_flush():69] Unhandled environment var: WANDB_WATCH
+2021-07-09 12:43:53,661 INFO    MainThread:33657 [wandb_setup.py:_flush():69] setting env: {'project': 'hf-flax-robert-base-mr', 'entity': 'nipunsadvilkar'}
+2021-07-09 12:43:53,661 INFO    MainThread:33657 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-07-09 12:43:53,662 INFO    MainThread:33657 [wandb_init.py:_log_setup():337] Logging user logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/logs/debug.log
+2021-07-09 12:43:53,662 INFO    MainThread:33657 [wandb_init.py:_log_setup():338] Logging internal logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_124353-3pjvcule/logs/debug-internal.log
+2021-07-09 12:43:53,662 INFO    MainThread:33657 [wandb_init.py:init():370] calling init triggers
+2021-07-09 12:43:53,662 INFO    MainThread:33657 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
+config: {}
+2021-07-09 12:43:53,662 INFO    MainThread:33657 [wandb_init.py:init():419] starting backend
+2021-07-09 12:43:53,662 INFO    MainThread:33657 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-07-09 12:43:53,696 INFO    MainThread:33657 [backend.py:ensure_launched():135] starting backend process...
+2021-07-09 12:43:53,729 INFO    MainThread:33657 [backend.py:ensure_launched():139] started backend process with pid: 34715
+2021-07-09 12:43:53,731 INFO    MainThread:33657 [wandb_init.py:init():424] backend started and connected
+2021-07-09 12:43:53,735 INFO    MainThread:33657 [wandb_init.py:init():472] updated telemetry
+2021-07-09 12:43:53,736 INFO    MainThread:33657 [wandb_init.py:init():491] communicating current version
+2021-07-09 12:43:54,402 INFO    MainThread:33657 [wandb_init.py:init():496] got version response 
+2021-07-09 12:43:54,402 INFO    MainThread:33657 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
+2021-07-09 12:43:54,471 INFO    MainThread:33657 [wandb_init.py:init():529] starting run threads in backend
+2021-07-09 12:43:54,529 INFO    MainThread:33657 [wandb_run.py:_console_start():1623] atexit reg
+2021-07-09 12:43:54,530 INFO    MainThread:33657 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
+2021-07-09 12:43:54,530 INFO    MainThread:33657 [wandb_run.py:_redirect():1502] Redirecting console.
+2021-07-09 12:43:54,533 INFO    MainThread:33657 [wandb_run.py:_redirect():1558] Redirects installed.
+2021-07-09 12:43:54,533 INFO    MainThread:33657 [wandb_init.py:init():554] run started, returning control to user process
+2021-07-09 12:43:54,534 INFO    MainThread:33657 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 4, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 8.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul09_12-43-47_t1v-n-112df4a9-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 500, 'save_total_limit': None, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
+2021-07-09 12:43:54,535 INFO    MainThread:33657 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'roberta', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
+2021-07-09 12:43:54,535 INFO    MainThread:33657 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': 'oscar', 'dataset_config_name': 'unshuffled_deduplicated_als', 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 128, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False}
+2021-07-09 12:44:09,645 INFO    MainThread:33657 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255
+2021-07-09 12:44:09,646 INFO    MainThread:33657 [wandb_run.py:_restore():1565] restore

wandb/run-20210709_124449-choidyfc/files/config.yaml

@@ -0,0 +1,305 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.10.33
+    framework: huggingface
+    huggingface_version: 4.9.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.8.10
+    t:
+      1:
+      - 11
+      2:
+      - 11
+      4: 3.8.10
+      5: 0.10.33
+      6: 4.9.0.dev0
+      8:
+      - 5
+adafactor:
+  desc: null
+  value: false
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+cache_dir:
+  desc: null
+  value: null
+config_name:
+  desc: null
+  value: ./
+dataloader_drop_last:
+  desc: null
+  value: false
+dataloader_num_workers:
+  desc: null
+  value: 0
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataset_config_name:
+  desc: null
+  value: unshuffled_deduplicated_als
+dataset_name:
+  desc: null
+  value: oscar
+ddp_find_unused_parameters:
+  desc: null
+  value: null
+debug:
+  desc: null
+  value: []
+deepspeed:
+  desc: null
+  value: null
+disable_tqdm:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: false
+do_predict:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+dtype:
+  desc: null
+  value: float32
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_steps:
+  desc: null
+  value: 500
+evaluation_strategy:
+  desc: null
+  value: IntervalStrategy.NO
+fp16:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+fp16_full_eval:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+gradient_accumulation_steps:
+  desc: null
+  value: 1
+greater_is_better:
+  desc: null
+  value: null
+group_by_length:
+  desc: null
+  value: false
+ignore_data_skip:
+  desc: null
+  value: false
+label_names:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+learning_rate:
+  desc: null
+  value: 0.0003
+length_column_name:
+  desc: null
+  value: length
+line_by_line:
+  desc: null
+  value: false
+load_best_model_at_end:
+  desc: null
+  value: false
+local_rank:
+  desc: null
+  value: -1
+log_level:
+  desc: null
+  value: -1
+log_level_replica:
+  desc: null
+  value: -1
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: ./runs/Jul09_12-44-43_t1v-n-112df4a9-w-0
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 500
+logging_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+lr_scheduler_type:
+  desc: null
+  value: SchedulerType.LINEAR
+max_grad_norm:
+  desc: null
+  value: 1.0
+max_seq_length:
+  desc: null
+  value: 128
+max_steps:
+  desc: null
+  value: -1
+metric_for_best_model:
+  desc: null
+  value: null
+mlm_probability:
+  desc: null
+  value: 0.15
+model_name_or_path:
+  desc: null
+  value: null
+model_type:
+  desc: null
+  value: roberta
+mp_parameters:
+  desc: null
+  value: ''
+no_cuda:
+  desc: null
+  value: false
+num_train_epochs:
+  desc: null
+  value: 8.0
+output_dir:
+  desc: null
+  value: ./
+overwrite_cache:
+  desc: null
+  value: false
+overwrite_output_dir:
+  desc: null
+  value: true
+pad_to_max_length:
+  desc: null
+  value: false
+past_index:
+  desc: null
+  value: -1
+per_device_eval_batch_size:
+  desc: null
+  value: 4
+per_device_train_batch_size:
+  desc: null
+  value: 4
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+prediction_loss_only:
+  desc: null
+  value: false
+preprocessing_num_workers:
+  desc: null
+  value: null
+push_to_hub:
+  desc: null
+  value: false
+push_to_hub_model_id:
+  desc: null
+  value: ''
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: null
+remove_unused_columns:
+  desc: null
+  value: true
+report_to:
+  desc: null
+  value:
+  - tensorboard
+  - wandb
+resume_from_checkpoint:
+  desc: null
+  value: null
+run_name:
+  desc: null
+  value: ./
+save_on_each_node:
+  desc: null
+  value: false
+save_steps:
+  desc: null
+  value: 500
+save_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+save_total_limit:
+  desc: null
+  value: null
+seed:
+  desc: null
+  value: 42
+sharded_ddp:
+  desc: null
+  value: []
+skip_memory_metrics:
+  desc: null
+  value: true
+tokenizer_name:
+  desc: null
+  value: ./
+tpu_metrics_debug:
+  desc: null
+  value: false
+tpu_num_cores:
+  desc: null
+  value: null
+train_file:
+  desc: null
+  value: null
+train_ref_file:
+  desc: null
+  value: null
+use_fast_tokenizer:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+validation_file:
+  desc: null
+  value: null
+validation_ref_file:
+  desc: null
+  value: null
+validation_split_percentage:
+  desc: null
+  value: 5
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 1000
+weight_decay:
+  desc: null
+  value: 0.0