Upload folder using huggingface_hub

.gitattributes

@@ -0,0 +1,9 @@
+data/datasets/local/open-asssistant-conversations/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/open-asssistant-conversations/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/open-asssistant-conversations/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/open-asssistant-conversations/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/open-asssistant-conversations/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/open-asssistant-conversations/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/open-asssistant-conversations/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/open-asssistant-conversations/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/open-asssistant-conversations/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text

data/datasets/local/open-asssistant-conversations/config.yml

@@ -0,0 +1,10 @@
+name: open-asssistant-conversations
+namespace: local
+settings:
+  preferred_embedding: gte-small
+  ui:
+    media_paths:
+      - text
+source:
+  source_name: huggingface
+  dataset_name: OpenAssistant/oasst1

data/datasets/local/open-asssistant-conversations/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2557dc647ff10b0396e9b40f24468f599661c664ff777c62647605503dea94dc
+size 42071787

data/datasets/local/open-asssistant-conversations/manifest.json

@@ -0,0 +1,118 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "data_schema": {
+    "fields": {
+      "message_id": {
+        "dtype": "string"
+      },
+      "parent_id": {
+        "dtype": "string"
+      },
+      "user_id": {
+        "dtype": "string"
+      },
+      "created_date": {
+        "dtype": "string"
+      },
+      "text": {
+        "dtype": "string"
+      },
+      "role": {
+        "dtype": "string"
+      },
+      "lang": {
+        "dtype": "string"
+      },
+      "review_count": {
+        "dtype": "int32"
+      },
+      "review_result": {
+        "dtype": "boolean"
+      },
+      "deleted": {
+        "dtype": "boolean"
+      },
+      "rank": {
+        "dtype": "int32"
+      },
+      "synthetic": {
+        "dtype": "boolean"
+      },
+      "model_name": {
+        "dtype": "string"
+      },
+      "detoxify": {
+        "fields": {
+          "toxicity": {
+            "dtype": "float64"
+          },
+          "severe_toxicity": {
+            "dtype": "float64"
+          },
+          "obscene": {
+            "dtype": "float64"
+          },
+          "identity_attack": {
+            "dtype": "float64"
+          },
+          "insult": {
+            "dtype": "float64"
+          },
+          "threat": {
+            "dtype": "float64"
+          },
+          "sexual_explicit": {
+            "dtype": "float64"
+          }
+        }
+      },
+      "message_tree_id": {
+        "dtype": "string"
+      },
+      "tree_state": {
+        "dtype": "string"
+      },
+      "emojis": {
+        "fields": {
+          "name": {
+            "repeated_field": {
+              "dtype": "string"
+            }
+          },
+          "count": {
+            "repeated_field": {
+              "dtype": "int32"
+            }
+          }
+        }
+      },
+      "labels": {
+        "fields": {
+          "name": {
+            "repeated_field": {
+              "dtype": "string"
+            }
+          },
+          "value": {
+            "repeated_field": {
+              "dtype": "float64"
+            }
+          },
+          "count": {
+            "repeated_field": {
+              "dtype": "int32"
+            }
+          }
+        }
+      },
+      "__hfsplit__": {
+        "dtype": "string"
+      },
+      "__rowid__": {
+        "dtype": "string"
+      }
+    }
+  }
+}

data/datasets/local/open-asssistant-conversations/settings.json

@@ -0,0 +1 @@
+{"ui": {"media_paths": [["text"]], "markdown_paths": []}, "preferred_embedding": "gte-small"}

data/datasets/local/open-asssistant-conversations/text/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ef42015e1cfa76fc929c0a8913911c765e871586af7eac6f42def6abbd856f5
+size 327991004

data/datasets/local/open-asssistant-conversations/text/gte-small/hnsw.lookup.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4370d3885f9dea3df44fc7e366069c83c93af3b068ed5a56eaa2ac442c4f502c
+size 6171229

data/datasets/local/open-asssistant-conversations/text/gte-small/signal_manifest.json

@@ -0,0 +1,35 @@
+{
+  "files": [],
+  "parquet_id": "gte-small(text)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "text": {
+        "fields": {
+          "gte-small": {
+            "repeated_field": {
+              "fields": {
+                "embedding": {
+                  "dtype": "embedding"
+                }
+              },
+              "dtype": "string_span"
+            },
+            "signal": {
+              "signal_name": "gte-small"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "gte-small"
+  },
+  "enriched_path": [
+    "text"
+  ],
+  "vector_store": "hnsw"
+}

data/datasets/local/open-asssistant-conversations/text/gte-small/spans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9b6962f47eefe550b314cdb4d6c6eb0811670f5f41d137b952fcc55e1d331cc
+size 5164058

data/datasets/local/open-asssistant-conversations/text/lang_detection/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a81583a4ea611107922c1da8b6948845994bdf2080fbc1751a71ea18046d4bd
+size 2980615

data/datasets/local/open-asssistant-conversations/text/lang_detection/signal_manifest.json

@@ -0,0 +1,31 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "lang_detection(text)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "text": {
+        "fields": {
+          "lang_detection": {
+            "dtype": "string",
+            "signal": {
+              "split_by_paragraph": false,
+              "signal_name": "lang_detection"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "text"
+  ]
+}

data/datasets/local/open-asssistant-conversations/text/near_dup/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43c99611fc94cdd4998e03f18f651fe2ea7b515a5780bbcb78baa2030a3b39b1
+size 3485154

data/datasets/local/open-asssistant-conversations/text/near_dup/signal_manifest.json

@@ -0,0 +1,36 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "near_dup(text)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "text": {
+        "fields": {
+          "near_dup": {
+            "fields": {
+              "cluster_id": {
+                "dtype": "uint32",
+                "categorical": true
+              }
+            },
+            "signal": {
+              "threshold": 0.75,
+              "signal_name": "near_dup"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.75,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "text"
+  ]
+}

data/datasets/local/open-asssistant-conversations/text/pii/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cc6966d1c3c262121fa6130ff54e4ba7431d89ae81dfbc9ef9025f31bf095be
+size 2953280

data/datasets/local/open-asssistant-conversations/text/pii/signal_manifest.json

@@ -0,0 +1,45 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "pii(text)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "text": {
+        "fields": {
+          "pii": {
+            "fields": {
+              "emails": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "ip_addresses": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "secrets": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              }
+            },
+            "signal": {
+              "signal_name": "pii"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "text"
+  ]
+}

data/datasets/local/open-asssistant-conversations/text/spacy_ner/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:971edbdc4bdfad768444691a2e276f3c69e99a9f5251168aaa7fd2a89a649043
+size 5955494

data/datasets/local/open-asssistant-conversations/text/spacy_ner/signal_manifest.json

@@ -0,0 +1,38 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "spacy_ner(text)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "text": {
+        "fields": {
+          "spacy_ner": {
+            "repeated_field": {
+              "fields": {
+                "label": {
+                  "dtype": "string"
+                }
+              },
+              "dtype": "string_span"
+            },
+            "signal": {
+              "model": "en_core_web_sm",
+              "signal_name": "spacy_ner"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "model": "en_core_web_sm",
+    "signal_name": "spacy_ner"
+  },
+  "enriched_path": [
+    "text"
+  ]
+}

data/datasets/local/open-asssistant-conversations/text/text_statistics/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1adb0c31662191820bcffbefd09c00ecdc101bbc41b9941179ff0b4fd78d11b
+size 3827236

data/datasets/local/open-asssistant-conversations/text/text_statistics/signal_manifest.json

@@ -0,0 +1,59 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "text_statistics(text)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "text": {
+        "fields": {
+          "text_statistics": {
+            "fields": {
+              "num_characters": {
+                "dtype": "int32"
+              },
+              "readability": {
+                "dtype": "float32"
+              },
+              "log(type_token_ratio)": {
+                "dtype": "float32"
+              },
+              "frac_non_ascii": {
+                "dtype": "float32",
+                "bins": [
+                  [
+                    "Low",
+                    null,
+                    0.15
+                  ],
+                  [
+                    "Medium",
+                    0.15,
+                    0.3
+                  ],
+                  [
+                    "High",
+                    0.3,
+                    null
+                  ]
+                ]
+              }
+            },
+            "signal": {
+              "signal_name": "text_statistics"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "text"
+  ]
+}