Upload folder using huggingface_hub

.gitattributes

@@ -2,3 +2,13 @@ data/datasets/lilac/pile-of-law-constitutions/data-00000-of-00001.parquet filter
 data/datasets/lilac/pile-of-law-constitutions/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/pile-of-law-constitutions/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/pile-of-law-constitutions/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/piqa/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/piqa/goal/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/piqa/goal/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/piqa/goal/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/piqa/sol1/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/piqa/sol1/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/piqa/sol1/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/piqa/sol2/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/piqa/sol2/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/piqa/sol2/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text

data/datasets/lilac/piqa/config.yml

@@ -0,0 +1,67 @@
+embeddings:
+- embedding: gte-small
+  path: goal
+- embedding: gte-small
+  path: sol2
+- embedding: gte-small
+  path: sol1
+name: piqa
+namespace: local
+settings:
+  preferred_embedding: gte-small
+  ui:
+    media_paths:
+    - sol1
+    - sol2
+    - goal
+signals:
+- path: sol1
+  signal:
+    signal_name: near_dup
+- path: sol1
+  signal:
+    signal_name: text_statistics
+- path: sol1
+  signal:
+    signal_name: pii
+- path: sol1
+  signal:
+    signal_name: lang_detection
+- path: sol1
+  signal:
+    signal_name: spacy_ner
+- path: sol2
+  signal:
+    signal_name: near_dup
+- path: sol2
+  signal:
+    signal_name: pii
+- path: sol2
+  signal:
+    signal_name: spacy_ner
+- path: sol2
+  signal:
+    signal_name: lang_detection
+- path: sol2
+  signal:
+    signal_name: text_statistics
+- path: goal
+  signal:
+    signal_name: near_dup
+- path: goal
+  signal:
+    signal_name: text_statistics
+- path: goal
+  signal:
+    signal_name: spacy_ner
+- path: goal
+  signal:
+    signal_name: lang_detection
+- path: goal
+  signal:
+    signal_name: pii
+source:
+  dataset_name: piqa
+  source_name: huggingface
+tags:
+- machine-learning

data/datasets/lilac/piqa/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1530d99cdc0988b355d3dcbcd6f7e29439e4048535b598eaf08f2c372d5a76a4
+size 4040510

data/datasets/lilac/piqa/goal/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e46018f32560457c6bb77c8cb6270be81c48730a37e640d70de62b5e75d9b0ad
+size 35437836

data/datasets/lilac/piqa/goal/gte-small/hnsw.lookup.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:151e20cb34bc6ff100936ca9da15e7443e8ecc23cf68aa10977f90a5f5e3802b
+size 1010253

data/datasets/lilac/piqa/goal/gte-small/signal_manifest.json

@@ -0,0 +1,35 @@
+{
+  "files": [],
+  "parquet_id": "gte-small(goal)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "goal": {
+        "fields": {
+          "gte-small": {
+            "repeated_field": {
+              "fields": {
+                "embedding": {
+                  "dtype": "embedding"
+                }
+              },
+              "dtype": "string_span"
+            },
+            "signal": {
+              "signal_name": "gte-small"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "gte-small"
+  },
+  "enriched_path": [
+    "goal"
+  ],
+  "vector_store": "hnsw"
+}

data/datasets/lilac/piqa/goal/gte-small/spans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af17a91bcfa9ef113f360a6a6a625741b81ba887a577393c23aff0621535b52b
+size 1009873

data/datasets/lilac/piqa/goal/lang_detection/signal_manifest.json

@@ -0,0 +1,31 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "lang_detection(goal)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "goal": {
+        "fields": {
+          "lang_detection": {
+            "dtype": "string",
+            "signal": {
+              "split_by_paragraph": false,
+              "signal_name": "lang_detection"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "goal"
+  ]
+}

data/datasets/lilac/piqa/goal/near_dup/signal_manifest.json

@@ -0,0 +1,36 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "near_dup(goal)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "goal": {
+        "fields": {
+          "near_dup": {
+            "fields": {
+              "cluster_id": {
+                "dtype": "uint32",
+                "categorical": true
+              }
+            },
+            "signal": {
+              "threshold": 0.85,
+              "signal_name": "near_dup"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "goal"
+  ]
+}

data/datasets/lilac/piqa/goal/pii/signal_manifest.json

@@ -0,0 +1,45 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "pii(goal)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "goal": {
+        "fields": {
+          "pii": {
+            "fields": {
+              "emails": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "ip_addresses": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "secrets": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              }
+            },
+            "signal": {
+              "signal_name": "pii"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "goal"
+  ]
+}

data/datasets/lilac/piqa/goal/spacy_ner/signal_manifest.json

@@ -0,0 +1,38 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "spacy_ner(goal)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "goal": {
+        "fields": {
+          "spacy_ner": {
+            "repeated_field": {
+              "fields": {
+                "label": {
+                  "dtype": "string"
+                }
+              },
+              "dtype": "string_span"
+            },
+            "signal": {
+              "model": "en_core_web_sm",
+              "signal_name": "spacy_ner"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "model": "en_core_web_sm",
+    "signal_name": "spacy_ner"
+  },
+  "enriched_path": [
+    "goal"
+  ]
+}

data/datasets/lilac/piqa/goal/text_statistics/signal_manifest.json

@@ -0,0 +1,59 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "text_statistics(goal)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "goal": {
+        "fields": {
+          "text_statistics": {
+            "fields": {
+              "num_characters": {
+                "dtype": "int32"
+              },
+              "readability": {
+                "dtype": "float32"
+              },
+              "log(type_token_ratio)": {
+                "dtype": "float32"
+              },
+              "frac_non_ascii": {
+                "dtype": "float32",
+                "bins": [
+                  [
+                    "Low",
+                    null,
+                    0.15
+                  ],
+                  [
+                    "Medium",
+                    0.15,
+                    0.3
+                  ],
+                  [
+                    "High",
+                    0.3,
+                    null
+                  ]
+                ]
+              }
+            },
+            "signal": {
+              "signal_name": "text_statistics"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "goal"
+  ]
+}

data/datasets/lilac/piqa/manifest.json

@@ -0,0 +1,27 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "data_schema": {
+    "fields": {
+      "goal": {
+        "dtype": "string"
+      },
+      "sol1": {
+        "dtype": "string"
+      },
+      "sol2": {
+        "dtype": "string"
+      },
+      "label": {
+        "dtype": "string"
+      },
+      "__hfsplit__": {
+        "dtype": "string"
+      },
+      "__rowid__": {
+        "dtype": "string"
+      }
+    }
+  }
+}

data/datasets/lilac/piqa/sol1/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91976ea520234c445b947e745fabddaae0cf455d2a8e8d18ccb945905c642ed2
+size 36174988

data/datasets/lilac/piqa/sol1/gte-small/hnsw.lookup.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1333c87108731a3851176d7751a918e8a170de5a6d4e0e0a5b79f8663b37494e
+size 1018110

data/datasets/lilac/piqa/sol1/gte-small/signal_manifest.json

@@ -0,0 +1,35 @@
+{
+  "files": [],
+  "parquet_id": "gte-small(sol1)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "sol1": {
+        "fields": {
+          "gte-small": {
+            "repeated_field": {
+              "fields": {
+                "embedding": {
+                  "dtype": "embedding"
+                }
+              },
+              "dtype": "string_span"
+            },
+            "signal": {
+              "signal_name": "gte-small"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "gte-small"
+  },
+  "enriched_path": [
+    "sol1"
+  ],
+  "vector_store": "hnsw"
+}

data/datasets/lilac/piqa/sol1/gte-small/spans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b043969052fae73d232c222d10362ce093ca26b9224b6ea14c87cf65affd04df
+size 1015221

data/datasets/lilac/piqa/sol1/lang_detection/signal_manifest.json

@@ -0,0 +1,31 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "lang_detection(sol1)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "sol1": {
+        "fields": {
+          "lang_detection": {
+            "dtype": "string",
+            "signal": {
+              "split_by_paragraph": false,
+              "signal_name": "lang_detection"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "sol1"
+  ]
+}

data/datasets/lilac/piqa/sol1/near_dup/signal_manifest.json

@@ -0,0 +1,36 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "near_dup(sol1)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "sol1": {
+        "fields": {
+          "near_dup": {
+            "fields": {
+              "cluster_id": {
+                "dtype": "uint32",
+                "categorical": true
+              }
+            },
+            "signal": {
+              "threshold": 0.85,
+              "signal_name": "near_dup"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "sol1"
+  ]
+}

data/datasets/lilac/piqa/sol1/pii/signal_manifest.json

@@ -0,0 +1,45 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "pii(sol1)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "sol1": {
+        "fields": {
+          "pii": {
+            "fields": {
+              "emails": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "ip_addresses": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "secrets": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              }
+            },
+            "signal": {
+              "signal_name": "pii"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "sol1"
+  ]
+}

data/datasets/lilac/piqa/sol1/spacy_ner/signal_manifest.json

@@ -0,0 +1,38 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "spacy_ner(sol1)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "sol1": {
+        "fields": {
+          "spacy_ner": {
+            "repeated_field": {
+              "fields": {
+                "label": {
+                  "dtype": "string"
+                }
+              },
+              "dtype": "string_span"
+            },
+            "signal": {
+              "model": "en_core_web_sm",
+              "signal_name": "spacy_ner"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "model": "en_core_web_sm",
+    "signal_name": "spacy_ner"
+  },
+  "enriched_path": [
+    "sol1"
+  ]
+}

data/datasets/lilac/piqa/sol1/text_statistics/signal_manifest.json

@@ -0,0 +1,59 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "text_statistics(sol1)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "sol1": {
+        "fields": {
+          "text_statistics": {
+            "fields": {
+              "num_characters": {
+                "dtype": "int32"
+              },
+              "readability": {
+                "dtype": "float32"
+              },
+              "log(type_token_ratio)": {
+                "dtype": "float32"
+              },
+              "frac_non_ascii": {
+                "dtype": "float32",
+                "bins": [
+                  [
+                    "Low",
+                    null,
+                    0.15
+                  ],
+                  [
+                    "Medium",
+                    0.15,
+                    0.3
+                  ],
+                  [
+                    "High",
+                    0.3,
+                    null
+                  ]
+                ]
+              }
+            },
+            "signal": {
+              "signal_name": "text_statistics"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "sol1"
+  ]
+}

data/datasets/lilac/piqa/sol2/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b61683bccd432e615553a1e508b236f265663e74ec67c60edb4f368e169d7b65
+size 36180028

data/datasets/lilac/piqa/sol2/gte-small/hnsw.lookup.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d814ae715f918b41488589fabed6fba617820b1619c484e437a6d575a89692c3
+size 1018164

data/datasets/lilac/piqa/sol2/gte-small/signal_manifest.json

@@ -0,0 +1,35 @@
+{
+  "files": [],
+  "parquet_id": "gte-small(sol2)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "sol2": {
+        "fields": {
+          "gte-small": {
+            "repeated_field": {
+              "fields": {
+                "embedding": {
+                  "dtype": "embedding"
+                }
+              },
+              "dtype": "string_span"
+            },
+            "signal": {
+              "signal_name": "gte-small"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "gte-small"
+  },
+  "enriched_path": [
+    "sol2"
+  ],
+  "vector_store": "hnsw"
+}

data/datasets/lilac/piqa/sol2/gte-small/spans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bae45d635be51218f7dbf110a5fd4935d7afb802dd79425d18164161e2f1645
+size 1015232

data/datasets/lilac/piqa/sol2/lang_detection/signal_manifest.json

@@ -0,0 +1,31 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "lang_detection(sol2)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "sol2": {
+        "fields": {
+          "lang_detection": {
+            "dtype": "string",
+            "signal": {
+              "split_by_paragraph": false,
+              "signal_name": "lang_detection"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "sol2"
+  ]
+}

data/datasets/lilac/piqa/sol2/near_dup/signal_manifest.json

@@ -0,0 +1,36 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "near_dup(sol2)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "sol2": {
+        "fields": {
+          "near_dup": {
+            "fields": {
+              "cluster_id": {
+                "dtype": "uint32",
+                "categorical": true
+              }
+            },
+            "signal": {
+              "threshold": 0.85,
+              "signal_name": "near_dup"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "sol2"
+  ]
+}

data/datasets/lilac/piqa/sol2/pii/signal_manifest.json

@@ -0,0 +1,45 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "pii(sol2)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "sol2": {
+        "fields": {
+          "pii": {
+            "fields": {
+              "emails": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "ip_addresses": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "secrets": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              }
+            },
+            "signal": {
+              "signal_name": "pii"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "sol2"
+  ]
+}

data/datasets/lilac/piqa/sol2/spacy_ner/signal_manifest.json

@@ -0,0 +1,38 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "spacy_ner(sol2)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "sol2": {
+        "fields": {
+          "spacy_ner": {
+            "repeated_field": {
+              "fields": {
+                "label": {
+                  "dtype": "string"
+                }
+              },
+              "dtype": "string_span"
+            },
+            "signal": {
+              "model": "en_core_web_sm",
+              "signal_name": "spacy_ner"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "model": "en_core_web_sm",
+    "signal_name": "spacy_ner"
+  },
+  "enriched_path": [
+    "sol2"
+  ]
+}

data/datasets/lilac/piqa/sol2/text_statistics/signal_manifest.json

@@ -0,0 +1,59 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "text_statistics(sol2)",
+  "data_schema": {
+    "fields": {
+      "__rowid__": {
+        "dtype": "string"
+      },
+      "sol2": {
+        "fields": {
+          "text_statistics": {
+            "fields": {
+              "num_characters": {
+                "dtype": "int32"
+              },
+              "readability": {
+                "dtype": "float32"
+              },
+              "log(type_token_ratio)": {
+                "dtype": "float32"
+              },
+              "frac_non_ascii": {
+                "dtype": "float32",
+                "bins": [
+                  [
+                    "Low",
+                    null,
+                    0.15
+                  ],
+                  [
+                    "Medium",
+                    0.15,
+                    0.3
+                  ],
+                  [
+                    "High",
+                    0.3,
+                    null
+                  ]
+                ]
+              }
+            },
+            "signal": {
+              "signal_name": "text_statistics"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "sol2"
+  ]
+}