Upload folder using huggingface_hub

.gitattributes

@@ -87,3 +87,11 @@ data/datasets/lilac/wikitext-2-raw-v1/text/lang_detection/data-00000-of-00001.pa
 data/datasets/lilac/wikitext-2-raw-v1/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/wikitext-2-raw-v1/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/wikitext-2-raw-v1/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/medical_dialog/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/medical_dialog/dialogue_turns/utterance/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/medical_dialog/dialogue_turns/utterance/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/medical_dialog/dialogue_turns/utterance/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/medical_dialog/dialogue_turns/utterance/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/medical_dialog/dialogue_turns/utterance/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/medical_dialog/dialogue_turns/utterance/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/medical_dialog/dialogue_turns/utterance/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text

data/datasets/lilac/medical_dialog/config.yml

@@ -0,0 +1,46 @@
+embeddings:
+- embedding: gte-small
+  path:
+  - dialogue_turns
+  - utterance
+  - '*'
+name: medical_dialog
+namespace: lilac
+settings:
+  preferred_embedding: gte-small
+  ui:
+    media_paths:
+    - - dialogue_turns
+      - utterance
+      - '*'
+signals:
+- path:
+  - dialogue_turns
+  - utterance
+  - '*'
+  signal:
+    signal_name: near_dup
+- path:
+  - dialogue_turns
+  - utterance
+  - '*'
+  signal:
+    signal_name: text_statistics
+- path:
+  - dialogue_turns
+  - utterance
+  - '*'
+  signal:
+    signal_name: pii
+- path:
+  - dialogue_turns
+  - utterance
+  - '*'
+  signal:
+    signal_name: lang_detection
+source:
+  config_name: en
+  dataset_name: medical_dialog
+  source_name: huggingface
+tags:
+- medical

data/datasets/lilac/medical_dialog/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91f506cf1e86dbf133857b13ccdbe1dbbb2c5e616f83197502790739257d2f06
+size 134394415

data/datasets/lilac/medical_dialog/dialogue_turns/utterance/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88b70375d7904939284f054ce7a116b86d670a9b467edf045e9a1263b8bc740c
+size 1271359144

data/datasets/lilac/medical_dialog/dialogue_turns/utterance/gte-small/hnsw.lookup.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b523a5b28a70edaa92063c5a151b887037fb8d14ca91583fd989c0484f8f1855
+size 20992247

data/datasets/lilac/medical_dialog/dialogue_turns/utterance/gte-small/signal_manifest.json

@@ -0,0 +1,40 @@
+{
+  "files": [],
+  "parquet_id": "dialogue_turns.utterance.gte-small",
+  "data_schema": {
+    "fields": {
+      "dialogue_turns": {
+        "fields": {
+          "utterance": {
+            "repeated_field": {
+              "fields": {
+                "gte-small": {
+                  "repeated_field": {
+                    "fields": {
+                      "embedding": {
+                        "dtype": "embedding"
+                      }
+                    },
+                    "dtype": "string_span"
+                  },
+                  "signal": {
+                    "signal_name": "gte-small"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "gte-small"
+  },
+  "enriched_path": [
+    "dialogue_turns",
+    "utterance",
+    "*"
+  ],
+  "vector_store": "hnsw"
+}

data/datasets/lilac/medical_dialog/dialogue_turns/utterance/gte-small/spans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50c7a00e5d244549fca543f29703a79d79b0be8dbd934e724020aa7acd5c7cf0
+size 17593826

data/datasets/lilac/medical_dialog/dialogue_turns/utterance/lang_detection/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c5263a90c6ab29d6b39d6a6b5393118ce58aef0d3c2d414301eb4e2f3f3c4e3
+size 6380428

data/datasets/lilac/medical_dialog/dialogue_turns/utterance/lang_detection/signal_manifest.json

@@ -0,0 +1,36 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "dialogue_turns.utterance.lang_detection",
+  "data_schema": {
+    "fields": {
+      "dialogue_turns": {
+        "fields": {
+          "utterance": {
+            "repeated_field": {
+              "fields": {
+                "lang_detection": {
+                  "dtype": "string",
+                  "signal": {
+                    "split_by_paragraph": false,
+                    "signal_name": "lang_detection"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "dialogue_turns",
+    "utterance",
+    "*"
+  ]
+}

data/datasets/lilac/medical_dialog/dialogue_turns/utterance/near_dup/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c08f38f01f169afac104875890e33f829f50d0a6582c1838681d8fc836a57a1
+size 8687974

data/datasets/lilac/medical_dialog/dialogue_turns/utterance/near_dup/signal_manifest.json

@@ -0,0 +1,41 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "dialogue_turns.utterance.near_dup",
+  "data_schema": {
+    "fields": {
+      "dialogue_turns": {
+        "fields": {
+          "utterance": {
+            "repeated_field": {
+              "fields": {
+                "near_dup": {
+                  "fields": {
+                    "cluster_id": {
+                      "dtype": "uint32",
+                      "categorical": true
+                    }
+                  },
+                  "signal": {
+                    "threshold": 0.85,
+                    "signal_name": "near_dup"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "dialogue_turns",
+    "utterance",
+    "*"
+  ]
+}

data/datasets/lilac/medical_dialog/dialogue_turns/utterance/pii/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28bcf251e88f21fefa5df0417cc9c706730e1b4777ac6591323f209611f37f0f
+size 6428720

data/datasets/lilac/medical_dialog/dialogue_turns/utterance/pii/signal_manifest.json

@@ -0,0 +1,50 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "dialogue_turns.utterance.pii",
+  "data_schema": {
+    "fields": {
+      "dialogue_turns": {
+        "fields": {
+          "utterance": {
+            "repeated_field": {
+              "fields": {
+                "pii": {
+                  "fields": {
+                    "emails": {
+                      "repeated_field": {
+                        "dtype": "string_span"
+                      }
+                    },
+                    "ip_addresses": {
+                      "repeated_field": {
+                        "dtype": "string_span"
+                      }
+                    },
+                    "secrets": {
+                      "repeated_field": {
+                        "dtype": "string_span"
+                      }
+                    }
+                  },
+                  "signal": {
+                    "signal_name": "pii"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "dialogue_turns",
+    "utterance",
+    "*"
+  ]
+}

data/datasets/lilac/medical_dialog/dialogue_turns/utterance/text_statistics/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a77052cab1e9dbb312c92d4e735aa3b18fd4706b0f3098d20f59b99ea8068e67
+size 9486961

data/datasets/lilac/medical_dialog/dialogue_turns/utterance/text_statistics/signal_manifest.json

@@ -0,0 +1,64 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "dialogue_turns.utterance.text_statistics",
+  "data_schema": {
+    "fields": {
+      "dialogue_turns": {
+        "fields": {
+          "utterance": {
+            "repeated_field": {
+              "fields": {
+                "text_statistics": {
+                  "fields": {
+                    "num_characters": {
+                      "dtype": "int32"
+                    },
+                    "readability": {
+                      "dtype": "float32"
+                    },
+                    "log(type_token_ratio)": {
+                      "dtype": "float32"
+                    },
+                    "frac_non_ascii": {
+                      "dtype": "float32",
+                      "bins": [
+                        [
+                          "Low",
+                          null,
+                          0.15
+                        ],
+                        [
+                          "Medium",
+                          0.15,
+                          0.3
+                        ],
+                        [
+                          "High",
+                          0.3,
+                          null
+                        ]
+                      ]
+                    }
+                  },
+                  "signal": {
+                    "signal_name": "text_statistics"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "dialogue_turns",
+    "utterance",
+    "*"
+  ]
+}

data/datasets/lilac/medical_dialog/manifest.json

@@ -0,0 +1,35 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "data_schema": {
+    "fields": {
+      "file_name": {
+        "dtype": "string"
+      },
+      "dialogue_id": {
+        "dtype": "int32"
+      },
+      "dialogue_url": {
+        "dtype": "string"
+      },
+      "dialogue_turns": {
+        "fields": {
+          "speaker": {
+            "repeated_field": {
+              "dtype": "int32"
+            }
+          },
+          "utterance": {
+            "repeated_field": {
+              "dtype": "string"
+            }
+          }
+        }
+      },
+      "__hfsplit__": {
+        "dtype": "string"
+      }
+    }
+  }
+}