Upload folder using huggingface_hub

.gitattributes

@@ -48,3 +48,11 @@ data/datasets/lilac/mmlu_professional_law/question/lang_detection/data-00000-of-
 data/datasets/lilac/mmlu_professional_law/question/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/mmlu_professional_law/question/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/mmlu_professional_law/question/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/pile-of-law-r-legaladvice/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/pile-of-law-r-legaladvice/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/pile-of-law-r-legaladvice/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/pile-of-law-r-legaladvice/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/pile-of-law-r-legaladvice/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text

data/datasets/lilac/pile-of-law-r-legaladvice/config.yml

@@ -0,0 +1,21 @@
+embeddings:
+- {embedding: gte-small, path: text}
+name: pile-of-law-r-legaladvice
+namespace: lilac
+settings:
+  preferred_embedding: gte-small
+  ui:
+    media_paths: [text]
+signals:
+- path: text
+  signal: {signal_name: near_dup}
+- path: text
+  signal: {signal_name: text_statistics}
+- path: text
+  signal: {signal_name: pii}
+- path: text
+  signal: {signal_name: lang_detection}
+- path: text
+  signal: {signal_name: spacy_ner}
+source: {config_name: r_legaladvice, dataset_name: pile-of-law/pile-of-law, source_name: huggingface}
+tags: [legal]

data/datasets/lilac/pile-of-law-r-legaladvice/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:315910747b04a2b504c0df875caf84812252602ea10a14c2049a45737a80db86
+size 190604294

data/datasets/lilac/pile-of-law-r-legaladvice/manifest.json

@@ -0,0 +1,27 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "data_schema": {
+    "fields": {
+      "text": {
+        "dtype": "string"
+      },
+      "created_timestamp": {
+        "dtype": "string"
+      },
+      "downloaded_timestamp": {
+        "dtype": "string"
+      },
+      "url": {
+        "dtype": "string"
+      },
+      "__hfsplit__": {
+        "dtype": "string"
+      },
+      "__rowid__": {
+        "dtype": "string"
+      }
+    }
+  }
+}

data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b613d5dd1584d6a5f8323d1464fa1f9d7a6a3cb8547963a8f3451afc56542a3c
+size 1922075524

data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/hnsw.lookup.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6ea206b1c4dba51a9e18f6db3822615cfe4c0ec435d3ce8c0e78ed4a604d11c
+size 24945492

data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/signal_manifest.json

@@ -0,0 +1,35 @@
+{
+  "files": [],
+  "parquet_id": "gte-small(text)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "text": {
+        "fields": {
+          "gte-small": {
+            "repeated_field": {
+              "fields": {
+                "embedding": {
+                  "dtype": "embedding"
+                }
+              },
+              "dtype": "string_span"
+            },
+            "signal": {
+              "signal_name": "gte-small"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "gte-small"
+  },
+  "enriched_path": [
+    "text"
+  ],
+  "vector_store": "hnsw"
+}

data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/spans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c233892a5f8fb35cdff7539bc657e15b91f91be9410c63e5a6dfc0aa627831ae
+size 15099068

data/datasets/lilac/pile-of-law-r-legaladvice/text/lang_detection/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f09e13588467643bc515248dc8b75003ecab9787b8d30b01959dd4f85d0a5eb0
+size 4827225

data/datasets/lilac/pile-of-law-r-legaladvice/text/lang_detection/signal_manifest.json

@@ -0,0 +1,31 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "lang_detection(text)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "text": {
+        "fields": {
+          "lang_detection": {
+            "dtype": "string",
+            "signal": {
+              "split_by_paragraph": false,
+              "signal_name": "lang_detection"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "text"
+  ]
+}

data/datasets/lilac/pile-of-law-r-legaladvice/text/near_dup/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3d7c4f4df146865336e8e0addba0ff1bfe0218bd6ef7d9b6cd2b1ef21a20405
+size 5743841

data/datasets/lilac/pile-of-law-r-legaladvice/text/near_dup/signal_manifest.json

@@ -0,0 +1,36 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "near_dup(text)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "text": {
+        "fields": {
+          "near_dup": {
+            "fields": {
+              "cluster_id": {
+                "dtype": "uint32",
+                "categorical": true
+              }
+            },
+            "signal": {
+              "threshold": 0.85,
+              "signal_name": "near_dup"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "text"
+  ]
+}

data/datasets/lilac/pile-of-law-r-legaladvice/text/pii/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38ca9bdf0f4a9dd4b0d1bc83f6f6b1446aa5b61c0566e1ffa165dac1565e3099
+size 4834322

data/datasets/lilac/pile-of-law-r-legaladvice/text/pii/signal_manifest.json

@@ -0,0 +1,45 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "pii(text)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "text": {
+        "fields": {
+          "pii": {
+            "fields": {
+              "emails": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "ip_addresses": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "secrets": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              }
+            },
+            "signal": {
+              "signal_name": "pii"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "text"
+  ]
+}

data/datasets/lilac/pile-of-law-r-legaladvice/text/text_statistics/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcfb0d11c300e2d97d10c57e63b777cb347936e24253758c716bed24e0b7a2e8
+size 6649363

data/datasets/lilac/pile-of-law-r-legaladvice/text/text_statistics/signal_manifest.json

@@ -0,0 +1,59 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "text_statistics(text)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "text": {
+        "fields": {
+          "text_statistics": {
+            "fields": {
+              "num_characters": {
+                "dtype": "int32"
+              },
+              "readability": {
+                "dtype": "float32"
+              },
+              "log(type_token_ratio)": {
+                "dtype": "float32"
+              },
+              "frac_non_ascii": {
+                "dtype": "float32",
+                "bins": [
+                  [
+                    "Low",
+                    null,
+                    0.15
+                  ],
+                  [
+                    "Medium",
+                    0.15,
+                    0.3
+                  ],
+                  [
+                    "High",
+                    0.3,
+                    null
+                  ]
+                ]
+              }
+            },
+            "signal": {
+              "signal_name": "text_statistics"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "text"
+  ]
+}