difftreelog
feat thread_enter/thread_exit bindins
in: master
5 files changed
Cargo.tomldiffbeforeafterboth--- a/Cargo.toml
+++ b/Cargo.toml
@@ -19,9 +19,7 @@
jrsonnet-stdlib = { path = "./crates/jrsonnet-stdlib", version = "0.5.0-pre96" }
jrsonnet-cli = { path = "./crates/jrsonnet-cli", version = "0.5.0-pre96" }
jrsonnet-types = { path = "./crates/jrsonnet-types", version = "0.5.0-pre96" }
-
-jrsonnet-gcmodule = "0.3.6"
-
+jrsonnet-gcmodule = { version = "0.3.7" }
# Diagnostics.
# hi-doc is my library, which handles text formatting very well, but isn't polished enough yet
# Previous implementation was based on annotate-snippets, which I don't like for many reasons.
bindings/c/libjsonnet.hdiffbeforeafterboth--- a/bindings/c/libjsonnet.h
+++ b/bindings/c/libjsonnet.h
@@ -1,22 +1,9 @@
-/*
-Copyright 2015 Google Inc. All rights reserved.
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
#ifndef LIB_JSONNET_H
#define LIB_JSONNET_H
#include <stddef.h>
-/** \file This file is a library interface for evaluating Jsonnet. It is written in C++ but exposes
+/** \file This file is a library interface for evaluating Jsonnet. It is written in Rust but exposes
* a C interface for easier wrapping by other languages. See \see jsonnet_lib_test.c for an example
* of using the library.
*/
@@ -28,10 +15,10 @@
*
* If this isn't the sae as jsonnet_version() then you've got a mismatched binary / header.
*/
-#define LIB_JSONNET_VERSION "v0.16.0"
+#define LIB_JSONNET_VERSION "v0.20.0"
/** Return the version string of the Jsonnet interpreter. Conforms to semantic versioning
- * http://semver.org/ If this does not match LIB_JSONNET_VERSION then there is a mismatch between
+ * https://semver.org/ If this does not match LIB_JSONNET_VERSION then there is a mismatch between
* header and compiled library.
*/
const char *jsonnet_version(void);
@@ -66,10 +53,12 @@
* process's CWD. This is necessary so that imports from the content of the imported file can
* be resolved correctly. Allocate memory with jsonnet_realloc. Only use when *success = 1.
* \param success Set this byref param to 1 to indicate success and 0 for failure.
- * \returns The content of the imported file, or an error message.
+ * \param buf Set this byref param to the content of the imported file, or an error message. Allocate memory with jsonnet_realloc. Do not include a null terminator byte.
+ * \param buflen Set this byref param to the length of the data returned in buf.
+ * \returns 0 to indicate success and 1 for failure. On success, the content is in *buf. On failure, an error message is in *buf.
*/
-typedef char *JsonnetImportCallback(void *ctx, const char *base, const char *rel, char **found_here,
- int *success);
+typedef int JsonnetImportCallback(void *ctx, const char *base, const char *rel,
+ char **found_here, char **buf, size_t *buflen);
/** An opaque type which can only be utilized via the jsonnet_json_* family of functions.
*/
@@ -82,7 +71,7 @@
/** If the value is a number, return 1 and store the number in out, otherwise return 0.
*/
int jsonnet_json_extract_number(struct JsonnetVm *vm, const struct JsonnetJsonValue *v,
- double *out);
+ double *out);
/** Return 0 if the value is false, 1 if it is true, and 2 if it is not a bool.
*/
@@ -117,7 +106,7 @@
/** Add v to the end of the array.
*/
void jsonnet_json_array_append(struct JsonnetVm *vm, struct JsonnetJsonValue *arr,
- struct JsonnetJsonValue *v);
+ struct JsonnetJsonValue *v);
/** Make a JsonnetJsonValue representing an object with the given number of fields.
*
@@ -130,7 +119,7 @@
* This replaces any previous binding of the field.
*/
void jsonnet_json_object_append(struct JsonnetVm *vm, struct JsonnetJsonValue *obj, const char *f,
- struct JsonnetJsonValue *v);
+ struct JsonnetJsonValue *v);
/** Clean up a JSON subtree.
*
@@ -151,8 +140,8 @@
* \returns The content of the imported file, or an error message.
*/
typedef struct JsonnetJsonValue *JsonnetNativeCallback(void *ctx,
- const struct JsonnetJsonValue *const *argv,
- int *success);
+ const struct JsonnetJsonValue *const *argv,
+ int *success);
/** Allocate, resize, or free a buffer. This will abort if the memory cannot be allocated. It will
* only return NULL if sz was zero.
@@ -181,7 +170,7 @@
* \param params NULL-terminated array of the names of the params. Must be valid identifiers.
*/
void jsonnet_native_callback(struct JsonnetVm *vm, const char *name, JsonnetNativeCallback *cb,
- void *ctx, const char *const *params);
+ void *ctx, const char *const *params);
/** Bind a Jsonnet external var to the given string.
*
@@ -236,7 +225,7 @@
* \returns Either JSON or the error message.
*/
char *jsonnet_evaluate_snippet(struct JsonnetVm *vm, const char *filename, const char *snippet,
- int *error);
+ int *error);
/** Evaluate a file containing Jsonnet code, return a number of named JSON files.
*
@@ -260,7 +249,7 @@
* \returns Either the error, or a sequence of strings separated by \0, terminated with \0\0.
*/
char *jsonnet_evaluate_snippet_multi(struct JsonnetVm *vm, const char *filename,
- const char *snippet, int *error);
+ const char *snippet, int *error);
/** Evaluate a file containing Jsonnet code, return a number of JSON files.
*
@@ -284,9 +273,46 @@
* \returns Either the error, or a sequence of strings separated by \0, terminated with \0\0.
*/
char *jsonnet_evaluate_snippet_stream(struct JsonnetVm *vm, const char *filename,
- const char *snippet, int *error);
+ const char *snippet, int *error);
/** Complement of \see jsonnet_vm_make. */
void jsonnet_destroy(struct JsonnetVm *vm);
-#endif // LIB_JSONNET_H
+/** Jrsonnet addition.
+ *
+ * In jrsonnet, vm state is bound to the thread, because interpreter
+ * also uses thread_local storage for some things (I.e GC).
+ *
+ * It makes it impossible to correctly use those bindings in golang,
+ * where developer has little control over goroutine scheduler.
+ *
+ * To make it work, jrsonnet provides methods to dump and restore thread
+ * state manually, making it possible to wire it with golang.
+ */
+struct JrThreadCTX;
+
+/** Dump current thread state, to be restored with
+ * jrsonnet_reenter_thread.
+ */
+struct JrThreadCTX *jrsonnet_exit_thread();
+/** Restore thread state, freeing JrThreadCTX.
+ */
+void jrsonnet_reenter_thread(struct JrThreadCTX *ctx);
+
+struct JrThreadId;
+
+/** Get current thread id (opaque pointer).
+ */
+struct JrThreadId* jrsonnet_thread_id();
+
+/** Compare two thread ids, it is not the same as a == b.
+ *
+ * \returns 1 if the same thread, 0 otherwise
+ */
+int jrsonnet_thread_id_compare(struct JrThreadId *a, struct JrThreadId *b);
+
+/** Free thread id value.
+ */
+void jrsonnet_thread_id_free(struct JrThreadId *id);
+
+#endif // LIB_JSONNET_H
bindings/jsonnet/Cargo.tomldiffbeforeafterboth--- a/bindings/jsonnet/Cargo.toml
+++ b/bindings/jsonnet/Cargo.toml
@@ -23,6 +23,7 @@
jrsonnet-parser.workspace = true
jrsonnet-stdlib.workspace = true
jrsonnet-gcmodule.workspace = true
+jrsonnet-interner.workspace = true
[lib]
name = "jsonnet"
bindings/jsonnet/src/lib.rsdiffbeforeafterboth--- a/bindings/jsonnet/src/lib.rs
+++ b/bindings/jsonnet/src/lib.rs
@@ -40,7 +40,7 @@
/// then there is a mismatch between header and compiled library.
#[no_mangle]
pub extern "C" fn jsonnet_version() -> &'static [u8; 8] {
- b"v0.19.1\0"
+ b"v0.20.0\0"
}
unsafe fn parse_path(input: &CStr) -> Cow<Path> {
crates/jrsonnet-interner/src/lib.rsdiffbeforeafterboth1#![deny(2 unsafe_op_in_unsafe_fn,3 clippy::missing_safety_doc,4 clippy::undocumented_unsafe_blocks5)]6#![warn(clippy::pedantic, clippy::nursery)]7#![allow(clippy::missing_const_for_fn)]8use std::{9 borrow::Cow,10 cell::RefCell,11 fmt::{self, Display},12 hash::{BuildHasherDefault, Hash, Hasher},13 ops::Deref,14 str,15};1617use hashbrown::{hash_map::RawEntryMut, HashMap};18use jrsonnet_gcmodule::Trace;19use rustc_hash::FxHasher;2021mod inner;22use inner::Inner;2324/// Interned string25///26/// Provides O(1) comparsions and hashing, cheap copy, and cheap conversion to [`IBytes`]27#[derive(Clone, PartialOrd, Ord, Eq)]28pub struct IStr(Inner);29impl Trace for IStr {30 fn is_type_tracked() -> bool {31 false32 }33}3435impl IStr {36 #[must_use]37 pub fn empty() -> Self {38 "".into()39 }40 #[must_use]41 pub fn as_str(&self) -> &str {42 self as &str43 }4445 #[must_use]46 pub fn cast_bytes(self) -> IBytes {47 IBytes(self.0.clone())48 }49}5051impl Deref for IStr {52 type Target = str;5354 fn deref(&self) -> &Self::Target {55 // SAFETY: Inner::check_utf8 is called on IStr construction, data is utf-856 unsafe { self.0.as_str_unchecked() }57 }58}5960impl PartialEq for IStr {61 fn eq(&self, other: &Self) -> bool {62 // all IStr should be inlined into same pool63 Inner::ptr_eq(&self.0, &other.0)64 }65}6667impl PartialEq<str> for IStr {68 fn eq(&self, other: &str) -> bool {69 self as &str == other70 }71}7273impl Hash for IStr {74 fn hash<H: Hasher>(&self, state: &mut H) {75 // IStr is always obtained from pool, where no string have duplicate, thus every unique string has unique address76 state.write_usize(Inner::as_ptr(&self.0).cast::<()>() as usize);77 }78}7980impl Drop for IStr {81 fn drop(&mut self) {82 maybe_unpool(&self.0);83 }84}8586impl fmt::Debug for IStr {87 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {88 fmt::Debug::fmt(self as &str, f)89 }90}9192impl Display for IStr {93 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {94 fmt::Display::fmt(self as &str, f)95 }96}9798/// Interned byte array99#[derive(Clone, PartialOrd, Ord, Eq)]100pub struct IBytes(Inner);101impl Trace for IBytes {102 fn is_type_tracked() -> bool {103 false104 }105}106107impl IBytes {108 #[must_use]109 pub fn cast_str(self) -> Option<IStr> {110 if Inner::check_utf8(&self.0) {111 Some(IStr(self.0.clone()))112 } else {113 None114 }115 }116 /// # Safety117 /// data should be valid utf8118 unsafe fn cast_str_unchecked(self) -> IStr {119 // SAFETY: data is utf8120 unsafe { Inner::assume_utf8(&self.0) };121 IStr(self.0.clone())122 }123124 #[must_use]125 pub fn as_slice(&self) -> &[u8] {126 self.0.as_slice()127 }128}129130impl Deref for IBytes {131 type Target = [u8];132133 fn deref(&self) -> &Self::Target {134 self.0.as_slice()135 }136}137138impl PartialEq for IBytes {139 fn eq(&self, other: &Self) -> bool {140 // all IStr should be inlined into same pool141 Inner::ptr_eq(&self.0, &other.0)142 }143}144145impl Hash for IBytes {146 fn hash<H: Hasher>(&self, state: &mut H) {147 // IBytes is always obtained from pool, where no string have duplicate, thus every unique string has unique address148 state.write_usize(Inner::as_ptr(&self.0).cast::<()>() as usize);149 }150}151152impl Drop for IBytes {153 fn drop(&mut self) {154 maybe_unpool(&self.0);155 }156}157158fn maybe_unpool(inner: &Inner) {159 #[cold]160 #[inline(never)]161 fn unpool(inner: &Inner) {162 // May fail on program termination163 let _ = POOL.try_with(|pool| {164 let mut pool = pool.borrow_mut();165166 if pool.remove(inner).is_none() {167 // On some platforms (i.e i686-windows), try_with will not fail after TLS168 // destructor is called, but instead re-initialize the TLS with the empty pool.169 // Allow non-pooled Drop in this case.170 // https://github.com/CertainLach/jrsonnet/issues/98#issuecomment-1591624016171 //172 // However, if pool is not empty, most likely this is issue #113, and then I don't173 // have any explainations for now.174 assert!(pool.is_empty(), "received an unpooled string not during the program termination, please write any info regarding this crash to https://github.com/CertainLach/jrsonnet/issues/113, thanks!");175 }176 });177 }178 // First reference - current object, second - POOL179 if Inner::strong_count(inner) <= 2 {180 unpool(inner);181 }182}183184impl fmt::Debug for IBytes {185 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {186 fmt::Debug::fmt(self as &[u8], f)187 }188}189190impl<'c> From<Cow<'c, str>> for IStr {191 fn from(v: Cow<'c, str>) -> Self {192 intern_str(&v)193 }194}195impl From<&str> for IStr {196 fn from(v: &str) -> Self {197 intern_str(v)198 }199}200impl From<String> for IStr {201 fn from(s: String) -> Self {202 s.as_str().into()203 }204}205impl From<&String> for IStr {206 fn from(s: &String) -> Self {207 s.as_str().into()208 }209}210impl From<char> for IStr {211 fn from(value: char) -> Self {212 let mut buf = [0; 5];213 Self::from(&*value.encode_utf8(&mut buf))214 }215}216impl From<&[u8]> for IBytes {217 fn from(v: &[u8]) -> Self {218 intern_bytes(v)219 }220}221222thread_local! {223 static POOL: RefCell<HashMap<Inner, (), BuildHasherDefault<FxHasher>>> = RefCell::new(HashMap::with_capacity_and_hasher(200, BuildHasherDefault::default()));224}225226#[must_use]227pub fn intern_bytes(bytes: &[u8]) -> IBytes {228 POOL.with(|pool| {229 let mut pool = pool.borrow_mut();230 let entry = pool.raw_entry_mut().from_key(bytes);231 match entry {232 RawEntryMut::Occupied(i) => IBytes(i.get_key_value().0.clone()),233 RawEntryMut::Vacant(e) => {234 let (k, ()) = e.insert(Inner::new_bytes(bytes), ());235 IBytes(k.clone())236 }237 }238 })239}240241#[must_use]242pub fn intern_str(str: &str) -> IStr {243 // SAFETY: Rust strings always utf8244 unsafe { intern_bytes(str.as_bytes()).cast_str_unchecked() }245}246247#[cfg(test)]248mod tests {249 use crate::IStr;250251 #[test]252 fn simple() {253 let a = IStr::from("a");254 let b = IStr::from("a");255256 assert_eq!(a.as_ptr(), b.as_ptr());257 }258}1#![deny(2 unsafe_op_in_unsafe_fn,3 clippy::missing_safety_doc,4 clippy::undocumented_unsafe_blocks5)]6#![warn(clippy::pedantic, clippy::nursery)]7#![allow(clippy::missing_const_for_fn)]8use std::{9 borrow::Cow,10 cell::RefCell,11 fmt::{self, Display},12 hash::{BuildHasherDefault, Hash, Hasher},13 ops::Deref,14 str,15};1617use hashbrown::{hash_map::RawEntryMut, HashMap};18use jrsonnet_gcmodule::Trace;19use rustc_hash::FxHasher;2021mod inner;22use inner::Inner;2324/// Interned string25///26/// Provides O(1) comparsions and hashing, cheap copy, and cheap conversion to [`IBytes`]27#[derive(Clone, PartialOrd, Ord, Eq)]28pub struct IStr(Inner);29impl Trace for IStr {30 fn is_type_tracked() -> bool {31 false32 }33}3435impl IStr {36 #[must_use]37 pub fn empty() -> Self {38 "".into()39 }40 #[must_use]41 pub fn as_str(&self) -> &str {42 self as &str43 }4445 #[must_use]46 pub fn cast_bytes(self) -> IBytes {47 IBytes(self.0.clone())48 }49}5051impl Deref for IStr {52 type Target = str;5354 fn deref(&self) -> &Self::Target {55 // SAFETY: Inner::check_utf8 is called on IStr construction, data is utf-856 unsafe { self.0.as_str_unchecked() }57 }58}5960impl PartialEq for IStr {61 fn eq(&self, other: &Self) -> bool {62 // all IStr should be inlined into same pool63 Inner::ptr_eq(&self.0, &other.0)64 }65}6667impl PartialEq<str> for IStr {68 fn eq(&self, other: &str) -> bool {69 self as &str == other70 }71}7273impl Hash for IStr {74 fn hash<H: Hasher>(&self, state: &mut H) {75 // IStr is always obtained from pool, where no string have duplicate, thus every unique string has unique address76 state.write_usize(Inner::as_ptr(&self.0).cast::<()>() as usize);77 }78}7980impl Drop for IStr {81 fn drop(&mut self) {82 maybe_unpool(&self.0);83 }84}8586impl fmt::Debug for IStr {87 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {88 fmt::Debug::fmt(self as &str, f)89 }90}9192impl Display for IStr {93 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {94 fmt::Display::fmt(self as &str, f)95 }96}9798/// Interned byte array99#[derive(Clone, PartialOrd, Ord, Eq)]100pub struct IBytes(Inner);101impl Trace for IBytes {102 fn is_type_tracked() -> bool {103 false104 }105}106107impl IBytes {108 #[must_use]109 pub fn cast_str(self) -> Option<IStr> {110 if Inner::check_utf8(&self.0) {111 Some(IStr(self.0.clone()))112 } else {113 None114 }115 }116 /// # Safety117 /// data should be valid utf8118 unsafe fn cast_str_unchecked(self) -> IStr {119 // SAFETY: data is utf8120 unsafe { Inner::assume_utf8(&self.0) };121 IStr(self.0.clone())122 }123124 #[must_use]125 pub fn as_slice(&self) -> &[u8] {126 self.0.as_slice()127 }128}129130impl Deref for IBytes {131 type Target = [u8];132133 fn deref(&self) -> &Self::Target {134 self.0.as_slice()135 }136}137138impl PartialEq for IBytes {139 fn eq(&self, other: &Self) -> bool {140 // all IStr should be inlined into same pool141 Inner::ptr_eq(&self.0, &other.0)142 }143}144145impl Hash for IBytes {146 fn hash<H: Hasher>(&self, state: &mut H) {147 // IBytes is always obtained from pool, where no string have duplicate, thus every unique string has unique address148 state.write_usize(Inner::as_ptr(&self.0).cast::<()>() as usize);149 }150}151152impl Drop for IBytes {153 fn drop(&mut self) {154 maybe_unpool(&self.0);155 }156}157158fn maybe_unpool(inner: &Inner) {159 #[cold]160 #[inline(never)]161 fn unpool(inner: &Inner) {162 // May fail on program termination163 let _ = POOL.try_with(|pool| {164 let mut pool = pool.borrow_mut();165166 if pool.remove(inner).is_none() {167 // On some platforms (i.e i686-windows), try_with will not fail after TLS168 // destructor is called, but instead re-initialize the TLS with the empty pool.169 // Allow non-pooled Drop in this case.170 // https://github.com/CertainLach/jrsonnet/issues/98#issuecomment-1591624016171 //172 // However, if pool is not empty, most likely this is issue #113, and then I don't173 // have any explainations for now.174 assert!(pool.is_empty(), "received an unpooled string not during the program termination, please write any info regarding this crash to https://github.com/CertainLach/jrsonnet/issues/113, thanks!");175 }176 });177 }178 // First reference - current object, second - POOL179 if Inner::strong_count(inner) <= 2 {180 unpool(inner);181 }182}183184impl fmt::Debug for IBytes {185 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {186 fmt::Debug::fmt(self as &[u8], f)187 }188}189190impl<'c> From<Cow<'c, str>> for IStr {191 fn from(v: Cow<'c, str>) -> Self {192 intern_str(&v)193 }194}195impl From<&str> for IStr {196 fn from(v: &str) -> Self {197 intern_str(v)198 }199}200impl From<String> for IStr {201 fn from(s: String) -> Self {202 s.as_str().into()203 }204}205impl From<&String> for IStr {206 fn from(s: &String) -> Self {207 s.as_str().into()208 }209}210impl From<char> for IStr {211 fn from(value: char) -> Self {212 let mut buf = [0; 5];213 Self::from(&*value.encode_utf8(&mut buf))214 }215}216impl From<&[u8]> for IBytes {217 fn from(v: &[u8]) -> Self {218 intern_bytes(v)219 }220}221222type PoolMap = HashMap<Inner, (), BuildHasherDefault<FxHasher>>;223224thread_local! {225 static POOL: RefCell<PoolMap> = RefCell::new(HashMap::with_capacity_and_hasher(200, BuildHasherDefault::default()));226}227228/// Jrsonnet golang bindings require that it is possible to move jsonnet229/// VM between OS threads, and this is not possible due to usage of230/// `thread_local`. Instead, there is two methods added, one should be231/// called at the end of current thread work, and one that should be232/// used when using other thread.233pub mod interop {234 use std::mem;235236 use crate::{PoolMap, POOL};237238 pub enum PoolState {}239240 /// Dump current interned string pool, to be restored by241 /// `reenter_thread`242 pub fn exit_thread() -> *mut PoolState {243 Box::into_raw(Box::new(POOL.with_borrow_mut(mem::take))).cast()244 }245246 /// Reenter thread, using state dumped by `exit_thread`.247 ///248 /// # Safety249 ///250 /// `state` should be acquired from `exit_thread`, it is not allowed251 /// to reuse state to reenter multiple threads.252 pub unsafe fn reenter_thread(state: *mut PoolState) {253 let ptr: *mut PoolMap = state.cast();254 // SAFETY: ptr is an unique state per method safety requirements.255 let ptr: Box<PoolMap> = unsafe { Box::from_raw(ptr) };256 let ptr: PoolMap = *ptr;257 POOL.with_borrow_mut(|pool| {258 let _ = mem::replace(pool, ptr);259 });260 }261}262263#[must_use]264pub fn intern_bytes(bytes: &[u8]) -> IBytes {265 POOL.with(|pool| {266 let mut pool = pool.borrow_mut();267 let entry = pool.raw_entry_mut().from_key(bytes);268 match entry {269 RawEntryMut::Occupied(i) => IBytes(i.get_key_value().0.clone()),270 RawEntryMut::Vacant(e) => {271 let (k, ()) = e.insert(Inner::new_bytes(bytes), ());272 IBytes(k.clone())273 }274 }275 })276}277278#[must_use]279pub fn intern_str(str: &str) -> IStr {280 // SAFETY: Rust strings always utf8281 unsafe { intern_bytes(str.as_bytes()).cast_str_unchecked() }282}283284#[cfg(test)]285mod tests {286 use crate::IStr;287288 #[test]289 fn simple() {290 let a = IStr::from("a");291 let b = IStr::from("a");292293 assert_eq!(a.as_ptr(), b.as_ptr());294 }295}