Upload folder using huggingface_hub

data/datasets/lilac/opus100-en-us-validation/config.yml

@@ -0,0 +1,26 @@
+name: opus100-en-us-validation
+namespace: lilac
+settings:
+  ui:
+    media_paths:
+    - [translation, es]
+    - [translation, en]
+signals:
+- path: [translation, es]
+  signal: {signal_name: near_dup}
+- path: [translation, es]
+  signal: {signal_name: pii}
+- path: [translation, es]
+  signal: {signal_name: lang_detection}
+- path: [translation, es]
+  signal: {signal_name: text_statistics}
+- path: [translation, en]
+  signal: {signal_name: near_dup}
+- path: [translation, en]
+  signal: {signal_name: text_statistics}
+- path: [translation, en]
+  signal: {signal_name: pii}
+- path: [translation, en]
+  signal: {signal_name: lang_detection}
+source: {config_name: en-es, dataset_name: opus100, source_name: huggingface, split: validation}
+tags: [machine-learning]

data/datasets/lilac/opus100-en-us-validation/manifest.json

@@ -0,0 +1,22 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "en": {
+            "dtype": "string"
+          },
+          "es": {
+            "dtype": "string"
+          }
+        }
+      },
+      "__hfsplit__": {
+        "dtype": "string"
+      }
+    }
+  }
+}

data/datasets/lilac/opus100-en-us-validation/translation/en/lang_detection/signal_manifest.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.en.lang_detection",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "en": {
+            "fields": {
+              "lang_detection": {
+                "dtype": "string",
+                "signal": {
+                  "split_by_paragraph": false,
+                  "signal_name": "lang_detection"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "translation",
+    "en"
+  ]
+}

data/datasets/lilac/opus100-en-us-validation/translation/en/near_dup/signal_manifest.json

@@ -0,0 +1,38 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.en.near_dup",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "en": {
+            "fields": {
+              "near_dup": {
+                "fields": {
+                  "cluster_id": {
+                    "dtype": "uint32",
+                    "categorical": true
+                  }
+                },
+                "signal": {
+                  "threshold": 0.85,
+                  "signal_name": "near_dup"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "translation",
+    "en"
+  ]
+}

data/datasets/lilac/opus100-en-us-validation/translation/en/pii/signal_manifest.json

@@ -0,0 +1,47 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.en.pii",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "en": {
+            "fields": {
+              "pii": {
+                "fields": {
+                  "emails": {
+                    "repeated_field": {
+                      "dtype": "string_span"
+                    }
+                  },
+                  "ip_addresses": {
+                    "repeated_field": {
+                      "dtype": "string_span"
+                    }
+                  },
+                  "secrets": {
+                    "repeated_field": {
+                      "dtype": "string_span"
+                    }
+                  }
+                },
+                "signal": {
+                  "signal_name": "pii"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "translation",
+    "en"
+  ]
+}

data/datasets/lilac/opus100-en-us-validation/translation/en/text_statistics/signal_manifest.json

@@ -0,0 +1,61 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.en.text_statistics",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "en": {
+            "fields": {
+              "text_statistics": {
+                "fields": {
+                  "num_characters": {
+                    "dtype": "int32"
+                  },
+                  "readability": {
+                    "dtype": "float32"
+                  },
+                  "log(type_token_ratio)": {
+                    "dtype": "float32"
+                  },
+                  "frac_non_ascii": {
+                    "dtype": "float32",
+                    "bins": [
+                      [
+                        "Low",
+                        null,
+                        0.15
+                      ],
+                      [
+                        "Medium",
+                        0.15,
+                        0.3
+                      ],
+                      [
+                        "High",
+                        0.3,
+                        null
+                      ]
+                    ]
+                  }
+                },
+                "signal": {
+                  "signal_name": "text_statistics"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "translation",
+    "en"
+  ]
+}

data/datasets/lilac/opus100-en-us-validation/translation/es/lang_detection/signal_manifest.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.es.lang_detection",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "es": {
+            "fields": {
+              "lang_detection": {
+                "dtype": "string",
+                "signal": {
+                  "split_by_paragraph": false,
+                  "signal_name": "lang_detection"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "translation",
+    "es"
+  ]
+}

data/datasets/lilac/opus100-en-us-validation/translation/es/near_dup/signal_manifest.json

@@ -0,0 +1,38 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.es.near_dup",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "es": {
+            "fields": {
+              "near_dup": {
+                "fields": {
+                  "cluster_id": {
+                    "dtype": "uint32",
+                    "categorical": true
+                  }
+                },
+                "signal": {
+                  "threshold": 0.85,
+                  "signal_name": "near_dup"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "translation",
+    "es"
+  ]
+}

data/datasets/lilac/opus100-en-us-validation/translation/es/pii/signal_manifest.json

@@ -0,0 +1,47 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.es.pii",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "es": {
+            "fields": {
+              "pii": {
+                "fields": {
+                  "emails": {
+                    "repeated_field": {
+                      "dtype": "string_span"
+                    }
+                  },
+                  "ip_addresses": {
+                    "repeated_field": {
+                      "dtype": "string_span"
+                    }
+                  },
+                  "secrets": {
+                    "repeated_field": {
+                      "dtype": "string_span"
+                    }
+                  }
+                },
+                "signal": {
+                  "signal_name": "pii"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "translation",
+    "es"
+  ]
+}

data/datasets/lilac/opus100-en-us-validation/translation/es/text_statistics/signal_manifest.json

@@ -0,0 +1,61 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.es.text_statistics",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "es": {
+            "fields": {
+              "text_statistics": {
+                "fields": {
+                  "num_characters": {
+                    "dtype": "int32"
+                  },
+                  "readability": {
+                    "dtype": "float32"
+                  },
+                  "log(type_token_ratio)": {
+                    "dtype": "float32"
+                  },
+                  "frac_non_ascii": {
+                    "dtype": "float32",
+                    "bins": [
+                      [
+                        "Low",
+                        null,
+                        0.15
+                      ],
+                      [
+                        "Medium",
+                        0.15,
+                        0.3
+                      ],
+                      [
+                        "High",
+                        0.3,
+                        null
+                      ]
+                    ]
+                  }
+                },
+                "signal": {
+                  "signal_name": "text_statistics"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "translation",
+    "es"
+  ]
+}