Upload folder using huggingface_hub

.gitattributes

@@ -26,3 +26,10 @@ data/datasets/lilac/piqa/sol1/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs
 data/datasets/lilac/piqa/sol2/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/piqa/sol2/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/piqa/sol2/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/enron-emails/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/enron-emails/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/enron-emails/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/enron-emails/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/enron-emails/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/enron-emails/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/enron-emails/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text

data/datasets/lilac/enron-emails/config.yml

@@ -0,0 +1,18 @@
+embeddings:
+- {embedding: gte-small, path: text}
+name: enron-emails
+namespace: lilac
+settings:
+  preferred_embedding: gte-small
+  ui:
+    media_paths: [text]
+signals:
+- path: text
+  signal: {signal_name: near_dup}
+- path: text
+  signal: {signal_name: lang_detection}
+- path: text
+  signal: {signal_name: pii}
+source: {config_name: enron_emails, dataset_name: EleutherAI/pile, sample_size: 100000,
+  source_name: huggingface}
+tags: [business]

data/datasets/lilac/enron-emails/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e9a6c560a9a4956d031d62b0ca7bb0832c270e1aa83114eab76685221946a2c
+size 164578444

data/datasets/lilac/enron-emails/manifest.json

@@ -0,0 +1,18 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "data_schema": {
+    "fields": {
+      "text": {
+        "dtype": "string"
+      },
+      "meta": {
+        "dtype": "string"
+      },
+      "__hfsplit__": {
+        "dtype": "string"
+      }
+    }
+  }
+}

data/datasets/lilac/enron-emails/text/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af45aa50af8fcc9c4a95b7558c39a19033e7481e55ae660b1b5addd2fd937998
+size 1912888576

data/datasets/lilac/enron-emails/text/gte-small/hnsw.lookup.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db97c5da2cc259cdb4410d0c8dfb30ae06dc3b98489d7041e62ac4350b2241f7
+size 23507190

data/datasets/lilac/enron-emails/text/gte-small/signal_manifest.json

@@ -0,0 +1,32 @@
+{
+  "files": [],
+  "parquet_id": "text.gte-small",
+  "data_schema": {
+    "fields": {
+      "text": {
+        "fields": {
+          "gte-small": {
+            "repeated_field": {
+              "fields": {
+                "embedding": {
+                  "dtype": "embedding"
+                }
+              },
+              "dtype": "string_span"
+            },
+            "signal": {
+              "signal_name": "gte-small"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "gte-small"
+  },
+  "enriched_path": [
+    "text"
+  ],
+  "vector_store": "hnsw"
+}

data/datasets/lilac/enron-emails/text/gte-small/spans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bc8ce021c57a9ee2c6548eda4d941b691b5604ddff277483dccccae35a6860b
+size 13603458

data/datasets/lilac/enron-emails/text/lang_detection/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2d2ee7230f452d35c24b842357274a63b70b3037cc91cfc86613bce169e2cbf
+size 3309649

data/datasets/lilac/enron-emails/text/lang_detection/signal_manifest.json

@@ -0,0 +1,28 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "text.lang_detection",
+  "data_schema": {
+    "fields": {
+      "text": {
+        "fields": {
+          "lang_detection": {
+            "dtype": "string",
+            "signal": {
+              "split_by_paragraph": false,
+              "signal_name": "lang_detection"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "text"
+  ]
+}

data/datasets/lilac/enron-emails/text/near_dup/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a12f04a049959c53570677eb42e60c63182d3f5dc8ab9a6619591418d813289
+size 3792281

data/datasets/lilac/enron-emails/text/near_dup/signal_manifest.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "text.near_dup",
+  "data_schema": {
+    "fields": {
+      "text": {
+        "fields": {
+          "near_dup": {
+            "fields": {
+              "cluster_id": {
+                "dtype": "uint32",
+                "categorical": true
+              }
+            },
+            "signal": {
+              "threshold": 0.85,
+              "signal_name": "near_dup"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "text"
+  ]
+}

data/datasets/lilac/enron-emails/text/pii/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32930c920525dcee9901aa16b5c7c427f95b955556e3a343059c6302cf86caa9
+size 10302224

data/datasets/lilac/enron-emails/text/pii/signal_manifest.json

@@ -0,0 +1,42 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "text.pii",
+  "data_schema": {
+    "fields": {
+      "text": {
+        "fields": {
+          "pii": {
+            "fields": {
+              "emails": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "ip_addresses": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "secrets": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              }
+            },
+            "signal": {
+              "signal_name": "pii"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "text"
+  ]
+}