Gabriele Sarti commited on
Commit
7b254d5
1 Parent(s): f01ac4b

Initial commit

Browse files
README.md CHANGED
@@ -1,3 +1,44 @@
1
  ---
2
- license: cc-by-nc-sa-4.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ language:
3
+ - en
4
+ - ja
5
+ tags:
6
+ - translation
7
+ - opus-mt-tc
8
+ license: cc-by-4.0
9
+ model-index:
10
+ - name: opus-mt-tc-base-en-ja
11
+ results:
12
+ - task:
13
+ name: Translation eng-jpg
14
+ type: translation
15
+ args: eng-jpg
16
+ dataset:
17
+ name: tatoeba-test-v2021-08-07
18
+ type: tatoeba_mt
19
+ args: eng-jpg
20
+ metrics:
21
+ - name: BLEU
22
+ type: bleu
23
+ value: 15.2
24
  ---
25
+
26
+ # Opus Tatoeba English-Japanese
27
+
28
+ *This model was obtained by running the script [convert_marian_to_pytorch.py](https://github.com/huggingface/transformers/blob/master/src/transformers/models/marian/convert_marian_to_pytorch.py) with the flag `-m eng-pol`. The original models were trained by [Jörg Tiedemann](https://blogs.helsinki.fi/tiedeman/) using the [MarianNMT](https://marian-nmt.github.io/) library. See all available `MarianMTModel` models on the profile of the [Helsinki NLP](https://huggingface.co/Helsinki-NLP) group.*
29
+
30
+ * dataset: opus+bt
31
+ * model: transformer-align
32
+ * source language(s): eng
33
+ * target language(s): jpn
34
+ * model: transformer-align
35
+ * pre-processing: normalization + SentencePiece (spm32k,spm32k)
36
+ * download: [opus+bt-2021-04-10.zip](https://object.pouta.csc.fi/Tatoeba-MT-models/eng-jpn/opus+bt-2021-04-10.zip)
37
+ * test set translations: [opus+bt-2021-04-10.test.txt](https://object.pouta.csc.fi/Tatoeba-MT-models/eng-jpn/opus+bt-2021-04-10.test.txt)
38
+ * test set scores: [opus+bt-2021-04-10.eval.txt](https://object.pouta.csc.fi/Tatoeba-MT-models/eng-jpn/opus+bt-2021-04-10.eval.txt)
39
+
40
+ ## Benchmarks
41
+
42
+ | testset | BLEU | chr-F | #sent | #words | BP |
43
+ |---------|-------|-------|-------|--------|----|
44
+ | Tatoeba-test.eng-jpn | 15.2 | 0.258 | 10000 | 99206 | 1.000 |
config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_dropout": 0.0,
3
+ "activation_function": "swish",
4
+ "architectures": [
5
+ "MarianMTModel"
6
+ ],
7
+ "attention_dropout": 0.0,
8
+ "bad_words_ids": [
9
+ [
10
+ 65000
11
+ ]
12
+ ],
13
+ "bos_token_id": 0,
14
+ "classifier_dropout": 0.0,
15
+ "d_model": 512,
16
+ "decoder_attention_heads": 8,
17
+ "decoder_ffn_dim": 2048,
18
+ "decoder_layerdrop": 0.0,
19
+ "decoder_layers": 6,
20
+ "decoder_start_token_id": 65000,
21
+ "decoder_vocab_size": 65001,
22
+ "dropout": 0.1,
23
+ "encoder_attention_heads": 8,
24
+ "encoder_ffn_dim": 2048,
25
+ "encoder_layerdrop": 0.0,
26
+ "encoder_layers": 6,
27
+ "eos_token_id": 0,
28
+ "forced_eos_token_id": 0,
29
+ "init_std": 0.02,
30
+ "is_encoder_decoder": true,
31
+ "max_length": 512,
32
+ "max_position_embeddings": 512,
33
+ "model_type": "marian",
34
+ "normalize_embedding": false,
35
+ "num_beams": 6,
36
+ "num_hidden_layers": 6,
37
+ "pad_token_id": 65000,
38
+ "scale_embedding": true,
39
+ "share_encoder_decoder_embeddings": true,
40
+ "static_position_embeddings": true,
41
+ "torch_dtype": "float16",
42
+ "transformers_version": "4.21.0.dev0",
43
+ "use_cache": true,
44
+ "vocab_size": 65001
45
+ }
opus+bt.spm32k-spm32k.transformer-align.valid1.log ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [2021-03-26 13:28:20] [valid] Ep. 1 : Up. 2500 : perplexity : 12.964 : new best
2
+ [2021-03-26 13:40:07] [valid] Ep. 1 : Up. 5000 : perplexity : 13.8338 : stalled 1 times (last best: 12.964)
3
+ [2021-03-26 13:51:55] [valid] Ep. 1 : Up. 7500 : perplexity : 14.2534 : stalled 2 times (last best: 12.964)
4
+ [2021-03-26 14:03:46] [valid] Ep. 1 : Up. 10000 : perplexity : 14.6025 : stalled 3 times (last best: 12.964)
5
+ [2021-03-26 14:15:34] [valid] Ep. 1 : Up. 12500 : perplexity : 14.756 : stalled 4 times (last best: 12.964)
6
+ [2021-03-26 14:27:21] [valid] Ep. 1 : Up. 15000 : perplexity : 15.0628 : stalled 5 times (last best: 12.964)
7
+ [2021-03-26 14:39:09] [valid] Ep. 1 : Up. 17500 : perplexity : 15.609 : stalled 6 times (last best: 12.964)
8
+ [2021-03-26 14:50:58] [valid] Ep. 1 : Up. 20000 : perplexity : 15.7898 : stalled 7 times (last best: 12.964)
9
+ [2021-03-26 15:02:46] [valid] Ep. 1 : Up. 22500 : perplexity : 15.7109 : stalled 8 times (last best: 12.964)
10
+ [2021-03-26 15:14:39] [valid] Ep. 1 : Up. 25000 : perplexity : 15.7126 : stalled 9 times (last best: 12.964)
11
+ [2021-03-26 15:26:43] [valid] Ep. 2 : Up. 27500 : perplexity : 15.5996 : stalled 10 times (last best: 12.964)
12
+ [2021-03-26 15:38:35] [valid] Ep. 2 : Up. 30000 : perplexity : 15.5451 : stalled 11 times (last best: 12.964)
13
+ [2021-03-26 15:50:25] [valid] Ep. 2 : Up. 32500 : perplexity : 15.4719 : stalled 12 times (last best: 12.964)
14
+ [2021-03-26 16:02:16] [valid] Ep. 2 : Up. 35000 : perplexity : 15.367 : stalled 13 times (last best: 12.964)
15
+ [2021-03-26 16:14:06] [valid] Ep. 2 : Up. 37500 : perplexity : 15.2384 : stalled 14 times (last best: 12.964)
16
+ [2021-03-26 16:25:56] [valid] Ep. 2 : Up. 40000 : perplexity : 15.1453 : stalled 15 times (last best: 12.964)
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5010d7e2ba0b133f0eb52a57780c4006ba266ae9f48a79daf7313cea52e4ab1
3
+ size 221610115
source.spm ADDED
Binary file (808 kB). View file
 
special_tokens_map.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "eos_token": "</s>",
3
+ "pad_token": "<pad>",
4
+ "unk_token": "<unk>"
5
+ }
target.spm ADDED
Binary file (834 kB). View file
 
tokenizer_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "eos_token": "</s>",
3
+ "model_max_length": 512,
4
+ "name_or_path": ".",
5
+ "pad_token": "<pad>",
6
+ "separate_vocabs": false,
7
+ "source_lang": "eng",
8
+ "sp_model_kwargs": {},
9
+ "special_tokens_map_file": null,
10
+ "target_lang": "jpn",
11
+ "tokenizer_class": "MarianTokenizer",
12
+ "tokenizer_file": null,
13
+ "unk_token": "<unk>"
14
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff