linkspace/

lib.rs

// Copyright Anton Sol
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
#![deny(missing_debug_implementations)]
#![allow(clippy::type_complexity)]
#![feature(
    array_chunks,
    slice_from_ptr_range,
    non_null_from_ref,
    split_array,
    cell_update,
    str_split_remainder,
    try_blocks,
    extend_one,
    slice_split_once,
    iterator_try_collect,
    thread_local,
    write_all_vectored,
    type_alias_impl_trait,
    concat_bytes,
    try_trait_v2
)]
#![doc = include_str!("../../../README.md")]
#![doc = include_str!("../README.md")]

/// The only error format linkspace supports - alias for anyhow::Error
pub type LkError = anyhow::Error;
/// Result for [LkError]
pub type LkResult<T = ()> = std::result::Result<T, LkError>;

/// Re-export common types
pub mod prelude {
    pub use super::*;
    pub use argument_traits::{LkEnv, Stmnts, TryAsSpace};
    pub use linkspace_core::{
        ShouldBreak,
        byte_fmt::{
            AB, B64,
            endian_types::{self, U64},
        },
        point::{
            Domain, Error as PointError, GroupID, Link, LkHash, LkIdentity, LkPath, LkPathArray,
            LkPathError, LkPathPart, LkPathSegm, MutXHeader, NewPoint, PFields, PFieldsExt, Point,
            PointArc, PointBox, PointExt, PointParts, PointPtr, PointTypeFlags, PubKey,
            RecvPointPtr, SetXHeader, SigningExt, Stamp, Tag, WithNewXHeader, XFlags, XHeader, ab,
            abf, as_abtxt_c,
            link::{link, linkf},
            lkpath, now,
            repr::PointFmt,
            try_ab,
        },
        predicate::exprs::{CoreTestOp, TestOp},
        query::options::KnownOptions,
        space_expr::Space,
    };
    pub use std::ops::ControlFlow;
}

use linkspace_core::point as lkp;
use prelude::*;

pub use prelude::LkIdentity;

pub use point::{lk_datapoint, lk_keypoint, lk_linkpoint};
/// creating points
pub mod point {
    use std::io;

    use anyhow::Context;
    use linkspace_core::{ShouldBreak, space_expr::Space};
    pub use lkp::CheckHash;
    use lkp::{PartialPoint, reroute::MutXHeaderPoint};

    use super::*;
    /**

    create a datapoint with upto MAX_CONTENT_SIZE bytes

    ```
    # use linkspace::{*,prelude::*,abe::*};
    # fn main() -> LkResult{
    let datap = lk_datapoint(b"Some data")?;
    assert_eq!(datap.hash().to_string(), "ATE6XG70_mx-kV2GCZUIkNijcPa8gd1-C3OYu1pXqcU");
    assert_eq!(datap.data() , b"Some data");
    # Ok(())}
    ```

    **/
    pub fn lk_datapoint(data: &[u8]) -> LkResult<PointBox> {
        lk_datapoint_ref(data).map(|v| v.as_box())
    }
    /// like [lk_datapoint] but keeps it on the stack in rust enum format.
    pub fn lk_datapoint_ref(data: &[u8]) -> LkResult<PointParts<'_>> {
        Ok(lkp::try_datapoint_ref(data)?)
    }
    /**

    create a new linkpoint [PointBox]
    ```
    # use linkspace::{*,prelude::{*,endian_types::*},abe::*};
    # fn main() -> LkResult{

    let datap = lk_datapoint(b"this is a datapoint")?;
    let path = lkpath(&[b"hello",b"world"]);
    let links = [
        Link{tag: ab(b"a datapoint"),ptr:datap.hash()},
        Link{tag: ab(b"another tag"),ptr:PUBLIC}
    ];
    let data = b"extra data for the linkpoint";
    let stamp = Some(U64::new(0)); // None == Some(now()).
    let linkpoint = lk_linkpoint(data,b"mydomain",&links)?;

    assert_eq!(linkpoint.hash().to_string(),"IdnnQjgxJLGxLZGKdaXWVxc82-U8KyJoyKK3sKlD8Lc");
    assert_eq!(linkpoint.data(), data);
    assert_eq!(*linkpoint.get_group(), PUBLIC);

    # Ok(())}

    ```
    **/
    pub fn lk_linkpoint(space: &dyn TryAsSpace, data: &[u8], links: &[Link]) -> LkResult<PointBox> {
        let Space {
            domain,
            group,
            path,
        } = space.try_as_space(&())?;
        lk_linkpoint_ref(domain, group, &path, data, links, lkp::now()).map(|v| v.as_box())
    }
    /// like [lk_linkpoint] but keeps it on the stack in rust enum format.
    pub fn lk_linkpoint_ref<'o>(
        domain: Domain,
        group: GroupID,
        path: &'o LkPath,
        data: &'o [u8],
        links: &'o [Link],
        stamp: Stamp,
    ) -> LkResult<PointParts<'o>> {
        Ok(lkp::try_linkpoint_ref(
            group, domain, path, links, data, stamp,
        )?)
    }
    /// create a keypoint, i.e. a signed [lk_linkpoint]
    pub fn lk_keypoint(space: &dyn TryAsSpace, data: &[u8], links: &[Link]) -> LkResult<PointBox> {
        let Space {
            domain,
            group,
            path,
        } = space.try_as_space(&())?;
        linkspace_core::thread_local::with_id(|key| {
            lk_keypoint_ref(key, domain, group, &path, data, links, lkp::now()).map(|v| v.as_box())
        })
        .context("No signing key has been set")?
    }
    /// like [lk_keypoint] but keeps it on the stack in rust enum format.
    pub fn lk_keypoint_ref<'o>(
        key: &LkIdentity,
        domain: Domain,
        group: GroupID,
        path: &'o LkPath,
        data: &'o [u8],
        links: &'o [Link],
        stamp: Stamp,
    ) -> LkResult<PointParts<'o>> {
        Ok(lkp::try_keypoint_ref(
            group, domain, path, links, data, stamp, key,
        )?)
    }
    /// utility function to build either [lk_keypoint] or [lk_linkpoint] depending on the optional key argument
    pub fn lk_point_ref<'o>(
        key: Option<&LkIdentity>,
        domain: Domain,
        group: GroupID,
        path: &'o LkPath,
        data: &'o [u8],
        links: &'o [Link],
        stamp: Stamp,
    ) -> LkResult<PointParts<'o>> {
        Ok(lkp::try_point(
            group, domain, path, links, data, stamp, key,
        )?)
    }

    pub fn lk_deserialize_ref<'a>(
        check: CheckHash,
        mut buf: &'a [u8],
        cb: &mut dyn FnMut(&PointPtr) -> ShouldBreak,
    ) -> Result<&'a [u8], PointError> {
        while !buf.is_empty() {
            let (p, rest) = lkp::deserialize::read_point(buf, check)?;
            buf = rest;
            if cb(&p) {
                break;
            };
        }
        Ok(buf)
    }
    pub fn lk_deserialize(buf: &[u8], mode: IOMode) -> Result<(PointBox, &[u8]), PointError> {
        let (mut head, inner, tail) = PartialPoint::take(buf, mode.check())?;
        mode.set_header(&mut head.xheader);
        let point = unsafe {
            PointBox::read_excl2(head, CheckHash::Check, |out| out.copy_from_slice(inner))?
        };
        mode.check_private(&point)?;
        Ok((point, tail))
    }
    pub fn lk_deserialize_arc(buf: &[u8], mode: IOMode) -> Result<(PointArc, &[u8]), PointError> {
        let (mut head, inner, tail) = PartialPoint::take(buf, mode.check())?;
        mode.set_header(&mut head.xheader);
        let point = unsafe {
            PointArc::read_excl(head, CheckHash::Check, |out| out.copy_from_slice(inner))?
        };
        mode.check_private(&point)?;
        Ok((point, tail))
    }

    /// Determine the length required for lk_deserialize to succeed - expects at least [consts::MIN_POINT_LEN] bytes
    pub fn lk_deserialize_len(buf: &[u8]) -> Result<u16, PointError> {
        lkp::deserialize::read_point_len(buf)
    }

    #[derive(Copy, Clone, Debug, PartialEq, Eq, Default)]
    #[cfg_attr(feature = "pyo3", pyo3::pyclass(frozen, eq))]
    #[cfg_attr(feature = "wasm", wasm_bindgen::prelude::wasm_bindgen)]
    /// Signals the source of a point.
    pub enum IOMode {
        #[default]
        Transmit,
        /// A trusted origin.
        /// Sets the XFlags::STORAGE bit, skips validating hashes, does not increment hop
        Storage,
    }
    impl IOMode {
        pub fn check_private(self, p: &impl Point) -> Result<(), PointError> {
            if self != IOMode::Storage {
                p.check_private()?
            }
            Ok(())
        }
        pub fn check(self) -> CheckHash {
            match self {
                IOMode::Transmit => CheckHash::Check,
                IOMode::Storage => CheckHash::SkipHash,
            }
        }
        pub fn set_header(self, h: &mut XHeader) {
            match self {
                IOMode::Transmit => {
                    h.incr_xhop();
                    h.xflags.remove(XFlags::STORAGE);
                }
                IOMode::Storage => {
                    h.xflags.insert(XFlags::STORAGE);
                }
            }
        }
    }

    /// Writes any form of [Point] into the binary point format
    /// When reading you need to indicate what xfields you're accepting with mask [XHeader::KEEP_COMMON] and friends
    /// Mode does additional checks - specifically in IOMode::Transmit it rejects PRIVATE group, increments hop, and clears XFlags::Storage.
    pub fn lk_serialize_ref(
        p: &dyn Point,
        mode: IOMode,
        out: &mut dyn FnMut(lkp::ByteSegments) -> LkResult<()>,
    ) -> LkResult<()> {
        let mut point = MutXHeaderPoint::new(p);
        match mode {
            IOMode::Transmit => {
                p.check_private().map_err(io::Error::other)?;
                point.xheader.incr_xhop();
                point.xheader.xflags.remove(XFlags::STORAGE);
            }
            IOMode::Storage => {
                point.xheader.xflags.insert(XFlags::STORAGE);
            }
        }
        out(point.byte_segments())
    }
    pub fn lk_serialize(p: &dyn Point) -> Box<[u8]> {
        let mut v = vec![].into_boxed_slice();
        lk_serialize_ref(p, IOMode::Transmit, &mut |o| {
            v = o.to_bytes();
            Ok(())
        })
        .unwrap();
        v
    }
    pub fn as_netarc(p: &dyn Point) -> PointArc {
        p.as_arc()
    }
    pub fn as_netbox(p: &dyn Point) -> PointBox {
        p.as_box()
    }
}

pub use query::{LkQuery, lkq_add, lkq_insert_mut, lkq_new, lkq_space};
/// [LkQuery] functions
pub mod query {
    /**
    A set of predicates and options used to select a range of packets

    The supported predicate fields are [PredicateType].
    The known options are [KnownOptions].

    ```
    # use linkspace::{*,prelude::*,abe::*, query::*};
    # fn main() -> LkResult{


    // Add multiple predicates and options at once.
    let query_str = r#"
      group = [#:pub]
      domain = [a:hello]
      prefix = /some/path
    "#;
    let mut query = lkq_new(&query_str,&())?;

    // the domain:group:prefix is so common it has an allias [lkq_space]

    // As these are AB-Expressions they are evaluated in the [UserScope] context
    lkq_add_mut(&mut query,"prefix = /some/[0]",&[b"path" as &[u8]])?;

    {
       // add the predicate that the stamp should hold a value less 100
       let mut query_with_stamp = lkq_add(&query, &"stamp < [u64:100]",&())?;
       // conflicting predicates are not allowed
       let result = lkq_add_mut(&mut query_with_stamp, &"stamp > [u64:100]",&());
       assert!( result.is_err());
    }

    // When adding only a single predicate you might be able to avoid an encoding step
    lkq_insert_mut(&mut query,"stamp","<", &*now())?;
    // Predicates get merged if they overlap
    lkq_insert_mut(&mut query,"stamp","<",&lka_eval("[now:-1D]",&())?)?;

    println!("{}",&lkq_stringify(&query,false));


    # Ok(()) }
    ```

    Unlike predicates, the order in which options are can matter.
    */
    #[derive(Clone, PartialEq, Default)]
    #[repr(transparent)]
    #[cfg_attr(feature = "pyo3", pyo3::pyclass(unsendable, eq))]
    #[cfg_attr(feature = "wasm", wasm_bindgen::prelude::wasm_bindgen)]
    pub struct LkQuery(pub(crate) linkspace_core::query::Query);

    /// An immutable empty_query
    pub static LK_Q: LkQuery = LkQuery(linkspace_core::query::Query::DEFAULT);

    impl std::fmt::Display for LkQuery {
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
            self.0.fmt(f, true)
        }
    }
    impl std::fmt::Debug for LkQuery {
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
            self.0.fmt(f, false)
        }
    }

    use anyhow::Context;
    pub use linkspace_core::predicate::predicate_type::PredicateType;
    pub use linkspace_core::prelude::KnownOptions;
    use linkspace_core::{prelude::ExtPredicate, query::Query};

    use super::*;
    /// Create a new [Query] from a set of statements. See [Query]
    pub fn lkq_new(stmnts: &dyn Stmnts, scope: &dyn LkEnv) -> LkResult<LkQuery> {
        let mut q = LK_Q.clone();
        lkq_add_mut(&mut q, stmnts, scope)?;
        Ok(q)
    }

    /// Create a new [LkQuery] specifically for matching a hash. Sets the right mode and limit count. See [system::lks_scan_hashes] if you don't care to watch the db.
    pub fn lkq_hash(hash: LkHash, stmnts: &dyn Stmnts) -> LkResult<LkQuery> {
        let mut q = LkQuery(linkspace_core::query::Query::hash_eq(hash));
        lkq_add_mut(&mut q, stmnts, &())?;
        if let Some(wopts) = &mut q.0.watch_options {
            wopts.id.extend(&now().0);
        }
        Ok(q)
    }
    /** Create a query form a space_expr i.e. domain:group:/path */
    /** The domain and group resolve to [crate::thread_local] variants if not supplied. i.e. "::/path" */
    /** The path matches the exact path.*/
    /** alternatively use "::/my/path//*"  for all matches, or you can be more specific such as "::/my/path//././?" for matches starting with /my/path and 4 or 5 parts. */ */
    pub fn lkq_space(
        space_expr: &str,
        stmnts: &dyn Stmnts,
        scope: &dyn LkEnv,
    ) -> LkResult<LkQuery> {
        let qexpr: linkspace_core::space_expr::ShortQueryExpr = space_expr.parse()?;
        let scope = scope.as_scope();
        let mut q = LkQuery(qexpr.eval(&scope.as_dyn())?);
        lkq_add_mut(&mut q, stmnts, &scope)?;
        Ok(q)
    }

    pub fn lkq_watch(mut query: LkQuery, watch_prefix: &[u8]) -> LkQuery {
        let wo = query.0.watch_options.get_or_insert(Default::default());
        wo.id = [watch_prefix, &wo.id].concat();
        query
    }
    pub fn lkq_path(space: &Space, pubkey: Option<PubKey>) -> LkQuery {
        let q = Query::subspace(
            Some(space.domain),
            Some(space.group),
            &space.path,
            true,
            pubkey,
        );
        LkQuery(q)
    }
    pub fn lkq_prefix(space: &Space, pubkey: Option<PubKey>) -> LkQuery {
        let q = Query::subspace(
            Some(space.domain),
            Some(space.group),
            &space.path,
            false,
            pubkey,
        );
        LkQuery(q)
    }

    /// Add a statement or option similar to [lkq_add_mut], but use three arguments and skip an evaluation step.
    /// I.e. lkq_add_mut(q,"{field} {op} {lka_encode(bytes)}") becomes lkq_insert_mut(q,field,op,bytes).
    ///
    /// Accepts options in the form ```lkq_insert_mut(q,"","get",b"log-asc")```.
    pub fn lkq_insert_mut(query: &mut LkQuery, field: &str, test: &str, val: &[u8]) -> LkResult {
        if field.is_empty() {
            query.0.add_option(test, &[val])
        } else {
            let epre = ExtPredicate {
                kind: field
                    .parse()
                    .with_context(|| format!("No such Field '{field}'"))?,
                op: test
                    .parse()
                    .map_err(|e| anyhow::anyhow!("bad operator `{e}`"))?,
                val: val.to_vec().into(),
            };
            query.0.predicates.add_ext_predicate(epre)
        }
    }
    /// Merge statements into a new query.
    pub fn lkq_add(query: &LkQuery, stmnts: &dyn Stmnts, scope: &dyn LkEnv) -> LkResult<LkQuery> {
        let mut query = query.clone();
        lkq_add_mut(&mut query, stmnts, scope)?;
        Ok(query)
    }

    /// Add multiple ABE encoded statements to a [Query]
    pub fn lkq_add_mut(query: &mut LkQuery, stmnts: &dyn Stmnts, scope: &dyn LkEnv) -> LkResult {
        let scope = scope.as_scope();
        let scope = scope.as_dyn();
        stmnts.per_line(&mut |line| query.0.parse(line, scope))
    }
    /// Clear a [Query] for reuse
    pub fn lkq_clear_mut(query: &mut LkQuery) {
        *query = LK_Q.clone();
    }
    /// Get the string representation of a [Query] - abe indicates wether to use \[\] expressions
    pub fn lkq_stringify(query: &LkQuery, abe: bool) -> String {
        query.0.to_str(abe)
    }
    /// If the query has any domain predicate return it - errors if it is a set of domains
    pub fn lkq_exact_domain(query: &LkQuery) -> LkResult<Option<Domain>> {
        let domain_set = query.0.predicates.domain;
        if domain_set == Default::default() {
            return Ok(None);
        }
        domain_set
            .as_eq()
            .map(Into::into)
            .context("query contains a set of domains")
            .map(Some)
    }
    /// If the query has any group predicate return it - errors if it is a set of groups
    pub fn lkq_exact_group(query: &LkQuery) -> LkResult<Option<GroupID>> {
        let group_set = query.0.predicates.group;
        if group_set == Default::default() {
            return Ok(None);
        }
        group_set
            .as_eq()
            .map(Into::into)
            .context("query contains a set of groups")
            .map(Some)
    }

    pub use linkspace_core::query::CompiledQuery;
    /// Compile a [Query] into a function that tests packets to deteremine if they match
    #[allow(clippy::type_complexity)]
    pub fn lkq_compile(q: LkQuery) -> LkResult<CompiledQuery> {
        q.0.compile()
    }
}

/// cryptographic key functions for use in [lk_keypoint]
pub mod identity {
    use super::prelude::*;
    pub use linkspace_commons_internal::argon2_identity::{
        self, Costs, DEFAULT_COST, INSECURE_COST, PARANOID_COST,
    };

    /// generate a new key
    pub fn lki_generate() -> LkIdentity {
        LkIdentity::generate()
    }
    /// Encrypt an identity using argon2d with the public key as the salt in URL_SAFE base64 (replaces "+/" with "-_")
    /// Empty passwords are always encrypted with the cheapest preset
    pub fn lki_encrypt(id: &LkIdentity, pass: &[u8]) -> String {
        argon2_identity::encrypt(
            id,
            pass,
            if pass.is_empty() {
                INSECURE_COST
            } else {
                DEFAULT_COST
            },
        )
    }
    /// NOTE: password length & complexity determines your security
    pub fn lki_encrypt_paranoid(id: &LkIdentity, pass: &[u8]) -> String {
        argon2_identity::encrypt(id, pass, PARANOID_COST)
    }
    /// decrypt the result of [lki_encrypt]
    pub fn lki_decrypt(enckey: &str, pass: &[u8]) -> LkResult<LkIdentity> {
        Ok(linkspace_commons_internal::argon2_identity::decrypt(
            enckey, pass,
        )?)
    }
    /// read the public key portion of a [lki_encrypt] string
    pub fn lki_pubkey(encrypted_identity: &str) -> LkResult<PubKey> {
        Ok(linkspace_commons_internal::argon2_identity::pubkey(
            encrypted_identity,
        )?)
    }
    /// Seed randomness to generate an adhoc ID usable without syncing.
    pub fn lki_adhoc(space: &dyn TryAsSpace, pass: &[u8]) -> LkResult<LkIdentity> {
        let space = space.try_as_space(&())?;
        Ok(linkspace_commons_internal::argon2_identity::adhoc(
            space.domain,
            space.group,
            pass,
        ))
    }
    #[doc(hidden)]
    pub use linkspace_commons_internal::argon2_identity::test_key;
}

#[cfg(feature = "lmdb")]
pub use system::lks_open;
#[cfg(feature = "inmem")]
pub use system::lks_open_inmem;
#[cfg(all(any(feature = "lmdb", feature = "inmem"), not(target_family = "wasm")))]
pub use system::lks_process_while;
#[cfg(any(feature = "lmdb", feature = "inmem"))]
pub use system::{
    LkSystem,
    cb::{on_point, on_point_then, try_on_point},
    lks_process, lks_save, lks_scan, lks_tap,
};

#[cfg(any(feature = "lmdb", feature = "inmem"))]
/// a system to watch for new points from other processes or threads
pub mod system {
    /**
    A linkspace system.
    The struct is not !Send. use [lks_open] in each thread.
    The first opened in each thread is set as the thread 'default'.

    It stores callbacks, a transaction to a storage engine, and receives/sends (inter-process) events of new writes.
    open/create with [lks_open] or [lks_open_inmem].
    Save packets with [lks_save]_*.
    Get packets with [lks_scan]_*, [lks_watch], and [lks_tap].
    Use [lks_process]_* to trigger watch and tap callbacks and to update the transaction.
    NOTE: Saving a packet will not show up in a scan before calling one of the [lks_process]* functions.

    To use multiple instances in a thread see [lks_chsys]
    **/

    #[derive(Clone, PartialEq)]
    #[repr(transparent)]
    #[cfg_attr(feature = "pyo3", pyo3::pyclass(unsendable, eq))]
    #[cfg_attr(feature = "wasm", wasm_bindgen::prelude::wasm_bindgen)]
    pub struct LkSystem(pub(crate) System);

    use std::cell::Cell;

    use crate::interop::sys_interop::System;
    use anyhow::Context;
    use linkspace_system::*;

    impl std::fmt::Debug for LkSystem {
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
            f.debug_struct("LkSystem")
                .field("info", &lks_info(Some(self)))
                .finish()
        }
    }

    use crate::query::LkQuery;
    use cb::PointHandler;
    use linkspace_core::{prelude::PredicateType, saturating_cast};
    use linkspace_system::thread_local::*;
    use tracing::debug_span;

    use super::*;
    /// open or create a [LkSystem].
    ///
    /// will look at `path` | $LKDB | '$HOME/lkdb'
    /// In the unlikely case you wish to open multiple storage engines, note that the first call (per thread) sets the instance used for [lka_eval] (see [varscope] for more info).
    #[cfg(feature = "lmdb")]
    pub fn lks_open(dir: impl AsRef<std::path::Path>) -> std::io::Result<LkSystem> {
        let rt = linkspace_system::open_linkspace_dir(dir.as_ref(), true)?;
        Ok(LkSystem(rt))
    }
    /// opens an in memory linkspace system.
    /// Unlike [lks_open] the data is not persistent, nor can it be shared across processes, but it can be [lks_split] and used across threads.
    #[cfg(feature = "inmem")]
    pub fn lks_open_inmem(name: &str) -> LkSystem {
        let rt = linkspace_system::open_inmem(name);
        LkSystem(rt)
    }
    #[cfg(feature = "inmem")]
    #[doc(hidden)]
    pub use linkspace_system::storage_engine::inmem::IN_MEMORY_DIRTY_XSTAMP;

    pub fn lks_from_info(lksi: &LksInfo) -> LkResult<LkSystem> {
        match lksi.kind.as_str() {
            #[cfg(feature = "lmdb")]
            "lmdb" => Ok(lks_open(&lksi.name)?),
            #[cfg(feature = "inmem")]
            "inmem" => Ok(lks_open_inmem(&lksi.name)),
            kind => anyhow::bail!("Not compiled with support for {kind} system"),
        }
    }
    /// create a clone without any callbacks. Usually to send to another thread. Requires its own lks_process calls.
    pub fn lks_split() -> LkSystem {
        LkSystem(get_system().unwrap().new_system())
    }

    /// get the current system - defaults to the first call to [lks_open*]
    pub fn lks_current() -> Option<crate::LkSystem> {
        linkspace_system::thread_local::opt_get_system().map(crate::LkSystem)
    }

    pub fn lks_chsys<A>(lks: &LkSystem, fnc: impl FnOnce() -> A) -> A {
        use_sys(&lks.0, fnc)
    }

    /// Last recv stamp of this system. i.e. last result from [lks_process]
    pub fn lks_last() -> Stamp {
        get_system().unwrap().last()
    }

    /// save a point - must [lks_process]_* for it to be show up in a [lks_scan]_*. Returns true if new and false if it is old.
    /// By convention if the current xstamp of a point equals zero, the database will copy the recv stamp into xstamp - This does not update the point provided as an argument.
    pub fn lks_save(point: &dyn Point) -> std::io::Result<bool> {
        save_dyn_one(get_system_io()?.storage_engine(), point).map(|o| o.is_written())
    }
    /// save multiple packets at once - returns the number of new packets written
    pub fn lks_save_all(ppoint: &[&dyn Point]) -> std::io::Result<usize> {
        let (start, excl) = lks_save_all_ext(ppoint)?;
        Ok((excl.get() - start.get()) as usize)
    }
    /// returns the range [incusive,exclusive) of recv stamps used to save new packets. total_new = r.1-r.0
    pub fn lks_save_all_ext(ppoint: &[&dyn Point]) -> std::io::Result<(Stamp, Stamp)> {
        let range =
            with_system(|lks| save_dyn_iter(lks.storage_engine(), ppoint.iter().copied(), |_| ()))?;
        Ok((range.start.into(), range.end.into()))
    }
    /// Calls on_new - after saving - for each new point. [Point::recv] will return the stamp as just set in the database.
    pub fn lks_save_new(
        ppoint: &[&dyn Point],
        on_new: &mut dyn FnMut(&dyn Point),
    ) -> std::io::Result<usize> {
        let range = with_system(|lks| {
            save_dyn_iter(lks.storage_engine(), ppoint.iter().copied(), |p| on_new(&p))
        })?;
        Ok(range.count())
    }
    /// Call the callback for every query matched packet in storage.
    /// Note that new packets are only readable after a call to [lks_process]_*.
    /// Unlike [lks_tap] this only checks the :scan / :linkscan options (it does nothing for :notify, or :watch)
    /// If no :scan is set assumes :scan space-desc
    /// Break early if the callback returns true.
    /// returns number of matches. Positive if cb returned true, otherwise negative
    pub fn lks_scan(
        query: &LkQuery,
        cb: &mut dyn for<'o> FnMut(&'o dyn Point) -> ShouldBreak,
    ) -> LkResult<i64> {
        let mut match_counter = 0; // excludes linkscan
        let options = query.0.get_scan_options();
        with_system(|lks| {
            let breaks = lks.scan_query(
                options,
                &query.0,
                &mut match_counter,
                None,
                &mut |point: &dyn Point, _wid: Option<&[u8]>, _: &System| {
                    cb::should_break(cb(point))
                },
            )?;
            Ok(linkspace_system::system::return_value(
                match_counter,
                breaks,
            ))
        })
    }

    /** read a single packet directly without copying.**/
    pub fn lks_scan_one_ref<A>(
        query: &LkQuery,
        cb: &mut dyn FnMut(&dyn Point) -> A,
    ) -> LkResult<Option<A>> {
        let mut result = None;
        let mut calls = |p: &dyn Point| {
            result = Some(cb(p));
            true
        };
        with_system(|lks| {
            let mode = query.0.get_scan_options();
            let reader = lks.dyn_reader();
            let _ = dyn_query(&*reader, 1, mode, &query.0.predicates, None, &mut calls);
        });
        Ok(result)
    }
    /** read a single packet directly without copying.**/
    pub fn lks_scan_one_hash_ref<A>(
        hash: LkHash,
        cb: &mut dyn FnMut(&dyn Point) -> A,
    ) -> LkResult<Option<A>> {
        with_system(|lks| {
            let mut result = None;
            lks.dyn_reader()
                .get_points_by_hash(&mut [hash].into_iter(), &mut |point| {
                    if let Ok(point) = point {
                        result = Some(cb(point));
                    }
                    true
                });
            Ok(result)
        })
    }

    /// TODO: DOC
    pub fn lks_scan_hashes<H: AsRef<LkHash>>(
        hashes: &mut dyn Iterator<Item = H>,
        cb: &mut dyn FnMut(Option<&dyn Point>, H) -> ShouldBreak,
    ) -> LkResult<i64> {
        let mut c = 0;
        with_system(|lks| {
            let r = lks.dyn_reader();
            let last_h: Cell<Option<H>> = Cell::new(None);
            let mut iter = std::iter::from_fn(|| {
                let item = hashes.next()?;
                let hash = *item.as_ref();
                last_h.set(Some(item));
                Some(hash)
            });
            let breaks = r.get_points_by_hash(&mut iter, &mut |result| {
                if result.is_ok() {
                    c += 1;
                }
                cb(result.ok(), last_h.take().unwrap())
            });
            Ok(saturating_cast(c, !breaks))
        })
    }

    /// [lks_tap] that ignores the :scan option, and inserts a :watch option if none are present.
    pub fn lks_watch(query: LkQuery, cb: impl PointHandler + 'static) -> LkResult<()> {
        lks_watch2(query, cb, debug_span!("lks_watch - (untraced)"))
    }

    /// [lks_watch] with a custom log [tracing::Span]
    /// The span will be entered on every callback.
    pub fn lks_watch2(
        mut query: LkQuery,
        cb: impl PointHandler + 'static,
        span: tracing::Span,
    ) -> LkResult<()> {
        with_system(|lks| {
            query.0.scan_options = None;
            query.0.watch_options.get_or_insert_default();
            lks.tap_query(query.0, interop::sys_interop::Handler(cb), span)?;
            Ok(())
        })
    }

    /**
    If the :scan option is set, cb for all matching packets.
    If the :watch option is set, watch for new matching packets.

    The absolute return value is the number of times the callback was called during :scan phase.
    A positive value means the callback finished by returning true|break - and won't register watch for further matches.

    A 0 or negative value means it is watching for new packets
    During [[lks_process]] or [[lks_process_while]] the cb is called for each matching packet.

    The watch is finished when
    - the cb returns true or [ControlFlow::Break].
    - the predicate set will never match again (e.g. the 'i_*' or 'recv' predicate shall never match again )
    - [[lks_stop]] is called with the matching watch id

    **/
    pub fn lks_tap(query: LkQuery, cb: impl PointHandler + 'static) -> LkResult<i64> {
        lks_tap2(query, cb, debug_span!("lks_tap - (untraced)"))
    }

    /// [lks_tap] with a custom log [tracing::Span]
    /// The span will be entered on every callback.
    pub fn lks_tap2(
        query: LkQuery,
        cb: impl PointHandler + 'static,
        span: tracing::Span,
    ) -> LkResult<i64> {
        with_system(|lks| lks.tap_query(query.0, interop::sys_interop::Handler(cb), span))
    }

    pub fn lks_multi_tap(
        queries: Vec<LkQuery>,
        mut cb: impl PointHandler + 'static,
        any_close: bool,
        span: tracing::Span,
    ) -> LkResult<i64> {
        with_system(|lks| {
            if queries.is_empty() {
                cb.on_close(&query::LK_Q, handlers::StopReason::Finish, 0, 0);
                return Ok(0);
            }
            let mq = handlers::MultiQueryHandler::new(
                interop::sys_interop::Handler(cb),
                queries.len(),
                any_close,
            );
            let mut total = 0;
            let mut has_neg = false;
            for q in queries {
                let span = tracing::debug_span!(parent: &span,"multi query", inner=?q);
                let r = lks.tap_query(q.0, mq.clone(), span)?;
                has_neg = has_neg || r < 0;
                total += r.abs();
            }

            Ok(total * if has_neg { -1 } else { 1 })
        })
    }

    /// See [lks_tap2]
    pub fn vspan(name: &str) -> tracing::Span {
        tracing::debug_span!("{}", name)
    }

    pub fn lks_forget_hashes(
        hashes: &[LkHash],
        cb: &mut dyn FnMut(&dyn Point),
    ) -> anyhow::Result<()> {
        lks_mutate(MutateOp::Forget, hashes, cb)
    }
    pub fn lks_forget(query: &LkQuery, cb: &mut dyn FnMut(&dyn Point)) -> LkResult<()> {
        let ro = query.0.get_scan_options();
        with_system(|lks| {
            let reader = lks.get_reader();
            // TODO - reader should lock _after_ write lock
            let mut hashes = get_query(&reader, ro, &query.0.predicates).map(|p| {
                tracing::info!(point=%PointFmt(&p), "Removing");
                p.hash()
            });
            storage_engine::methods::remove_by_hash(
                lks.storage_engine(),
                MutateOp::Forget,
                &mut hashes,
                cb,
            )?;
            Ok(())
        })
    }
    pub use linkspace_system::storage_engine::tables_traits::MutateOp;
    pub fn lks_mutate(
        op: MutateOp,
        hashes: &[LkHash],
        cb: &mut dyn FnMut(&dyn Point),
    ) -> anyhow::Result<()> {
        with_system(|lks| {
            Ok(storage_engine::methods::remove_by_hash(
                lks.storage_engine(),
                op,
                &mut hashes.iter().copied(),
                cb,
            )?)
        })
    }
    /// close lks_tap watches based on the query id ':watch ID' in the query.
    pub fn lks_stop(id: &[u8], range: bool) {
        with_system(|lks| {
            if range {
                lks.close_prefix(id)
            } else {
                lks.close(id)
            }
        })
    }
    pub fn lks_stop_query(query: &LkQuery, range: bool) -> LkResult<()> {
        let id = query
            .0
            .watch_id()
            .context("this query doesnt have a :watch ID")?;
        with_system(|lks| {
            if range {
                lks.close_prefix(id)
            } else {
                lks.close(id)
            }
        });
        Ok(())
    }
    /// process the log of new packets and trigger callbacks. Updates the reader to the latest state.
    pub fn lks_process() -> Stamp {
        with_system(|lks| lks.process())
    }
    /**
    process callbacks until either:

    The timeout has elapsed (0)
    if A watch id is given:
      => The watch was matched and the cb called: Positive if finished, negative if still active
    if No watch id is given:
      => # of new points processed
    **/
    #[cfg(not(target_family = "wasm"))]
    pub fn lks_process_while(
        watch: Option<&linkspace_core::query::WatchIDRef>,
        timeout: Stamp,
    ) -> LkResult<isize> {
        let timeout = (timeout != Stamp::ZERO)
            .then(|| std::time::Instant::now() + std::time::Duration::from_micros(timeout.get()));
        _lks_process_while(watch, timeout).map(|i| i.as_isize())
    }

    #[doc(hidden)]
    pub use linkspace_system::system::ProcessWhileResult;
    #[doc(hidden)]
    #[cfg(not(target_family = "wasm"))]
    // simplifies python ffi bindings
    pub fn _lks_process_while(
        watch: Option<&linkspace_core::query::WatchIDRef>,
        timeout: Option<std::time::Instant>,
    ) -> LkResult<ProcessWhileResult> {
        with_system(|lks| lks.run_while(timeout, watch))
    }

    /**
    A future awaiting new [lks_save] to the storage engine.
    **/
    pub async fn lks_await_write() -> Stamp {
        let lks = get_system().unwrap();
        let result = lks.await_write().await;
        unsafe { set_system(lks) };
        result
    }
    /** an alias for `while(true) {lks_process(lks); lks_await_write(lks).await }`
    Useful in an async context where other routines tap the system.
     **/
    pub async fn lks_process_async() {
        let lks = get_system().unwrap();
        loop {
            lks.process();
            lks.await_write().await;
        }
    }
    /// iterate over all active (WatchID,Query, test result) - test result does not check for limits and recv bounds
    pub fn lks_list_watches(
        cb: &mut dyn FnMut(&[u8], &LkQuery, Result<(), PredicateType>),
        point: Option<&dyn Point>,
    ) {
        with_system(|lks| {
            for el in lks.dbg_watches().entries() {
                let mut is_match = Ok(());
                if let Some(p) = point {
                    is_match = el.info.tests.test_point(p);
                }
                cb(
                    &el.info.query_id,
                    LkQuery::from_impl(&el.info.query),
                    is_match,
                )
            }
        })
    }

    #[cfg_attr(feature = "pyo3", pyo3::pyclass(frozen, get_all))]
    #[cfg_attr(
        feature = "wasm",
        wasm_bindgen::prelude::wasm_bindgen(getter_with_clone, inspectable)
    )]
    #[derive(Debug, Clone)]
    /// miscellaneous information about the system
    pub struct LksInfo {
        /// the kind of system
        pub kind: String,
        /// if applicable - a path or name for the specific instance
        pub name: String,
        pub instance: usize,
    }
    impl std::fmt::Display for LksInfo {
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
            std::fmt::Debug::fmt(self, f)
        }
    }
    /// get [LkInfo] of a linkspace system
    pub fn lks_info(lks: Option<&LkSystem>) -> LkResult<LksInfo> {
        let lks = lks
            .map(|o| o.0.clone())
            .unwrap_or_else(|| get_system().unwrap());
        let instance = lks.instance();
        let se = lks.storage_engine();
        Ok(LksInfo {
            instance,
            // required utf8 name to use in LkEnvDataSync
            name: se
                .dir()
                .to_str()
                .context("TODO: non-utf8 name not supported ATM")?
                .to_string(),
            kind: match se {
                #[cfg(feature = "lmdb")]
                linkspace_system::storage_engine::impl_enum::StorageEngineImplEnum::Lmdb(_) => {
                    "lmdb".to_string()
                }
                #[cfg(feature = "inmem")]
                linkspace_system::storage_engine::impl_enum::StorageEngineImplEnum::InMem(_) => {
                    "inmem".to_string()
                }
            },
        })
    }

    pub fn _lks_dump() {
        let lks = get_system().unwrap();
        let se = lks.storage_engine();
        storage_engine::impl_enum::debug_se(se);
    }

    #[cfg(any(feature = "lmdb", feature = "inmem"))]
    /** [lks_tap] takes a [PointHandler].
     **/
    pub mod cb {
        pub use linkspace_system::handlers::StopReason;

        use std::ops::{ControlFlow, Try};

        use linkspace_core::{ShouldBreak, point::Point, query::WatchIDRef};

        use super::query::LkQuery;

        pub fn should_break(o: ShouldBreak) -> ControlFlow<()> {
            if o {
                ControlFlow::Break(())
            } else {
                ControlFlow::Continue(())
            }
        }

        /// Callbacks stored in a [LkSystem] instance.
        /// Is implemented for `|p:&dyn Point, watch:Option<&WatchIDRef>, lks: &LkSystem| -> ControlFlow<()> {}`
        pub trait PointHandler {
            /// Handles matching points.
            fn on_point(
                &mut self,
                point: &dyn Point,
                watch: Option<&WatchIDRef>,
            ) -> ControlFlow<()>;
            /// Called when break, finished, or replaced
            fn on_close(
                &mut self,
                _: &LkQuery,
                _: StopReason,
                _total_calls: u64,
                _watch_calls: u64,
            ) {
            }
        }
        impl PointHandler for Box<dyn PointHandler> {
            fn on_point(
                &mut self,
                point: &dyn Point,
                watch: Option<&WatchIDRef>,
            ) -> ControlFlow<()> {
                (**self).on_point(point, watch)
            }

            fn on_close(
                &mut self,
                query: &LkQuery,
                reason: StopReason,
                total: u64,
                watch_matches: u64,
            ) {
                (**self).on_close(query, reason, total, watch_matches)
            }
        }
        impl<T: PointHandler> PointHandler for std::rc::Rc<std::cell::RefCell<T>> {
            fn on_point(
                &mut self,
                point: &dyn Point,
                watch: Option<&WatchIDRef>,
            ) -> ControlFlow<()> {
                self.borrow_mut().on_point(point, watch)
            }

            fn on_close(
                &mut self,
                q: &LkQuery,
                stop_reason: StopReason,
                total_calls: u64,
                watch_calls: u64,
            ) {
                self.borrow_mut()
                    .on_close(q, stop_reason, total_calls, watch_calls);
            }
        }

        #[derive(Copy, Clone, Debug)]
        pub struct Handle<
            H: FnMut(&dyn Point, Option<&WatchIDRef>) -> ControlFlow<()>,
            S: FnMut(&LkQuery, StopReason, u64, u64),
        > {
            pub on_point: H,
            pub on_close: S,
        }
        impl<H, S> PointHandler for Handle<H, S>
        where
            H: FnMut(&dyn Point, Option<&WatchIDRef>) -> ControlFlow<()>,
            S: FnMut(&LkQuery, StopReason, u64, u64),
        {
            fn on_point(
                &mut self,
                point: &dyn Point,
                watch: Option<&WatchIDRef>,
            ) -> ControlFlow<()> {
                let h: &mut H = &mut self.on_point;
                h(point, watch)
            }

            fn on_close(
                &mut self,
                query: &LkQuery,
                reason: StopReason,
                total: u64,
                watch_matches: u64,
            ) {
                let s: &mut S = &mut self.on_close;
                s(query, reason, total, watch_matches)
            }
        }
        pub fn on_point_then(
            mut handle_point: impl FnMut(&dyn Point) -> crate::ShouldBreak,
            mut on_close: impl FnMut(StopReason),
        ) -> impl PointHandler {
            Handle {
                on_point: move |point: &dyn Point, _watch: Option<&WatchIDRef>| {
                    if (handle_point)(point) {
                        ControlFlow::Break(())
                    } else {
                        ControlFlow::Continue(())
                    }
                },
                on_close: move |_, r, _, _| (on_close)(r),
            }
        }

        #[derive(Copy, Clone)]
        struct OnPoint<A>(A);
        /// takes a `fn(&dyn Point,Option<&[u8]>, &Linkspace) -> ShouldBreak` and returns impl [PointHandler]
        pub fn on_point(
            mut handle_point: impl FnMut(&dyn Point) -> crate::ShouldBreak,
        ) -> impl PointHandler {
            OnPoint(move |point: &dyn Point, _watch: Option<&WatchIDRef>| {
                if (handle_point)(point) {
                    ControlFlow::Break(())
                } else {
                    ControlFlow::Continue(())
                }
            })
        }

        /// takes any fn(&dyn Point,&Linkspace) -> Try (e.g. Result or Option) and returns impl [PointHandler] that logs on break
        pub fn try_on_point<F, E>(mut handle_point: F) -> impl PointHandler
        where
            E: std::fmt::Debug,
            F: for<'a> FnMut(&'a dyn Point) -> Result<(), E> + 'static,
        {
            OnPoint(move |point: &dyn Point, _watch: Option<&WatchIDRef>| {
                (handle_point)(point)
                    .branch()
                    .map_break(|brk| tracing::error!(?brk, "break"))
            })
        }

        impl<A> PointHandler for OnPoint<A>
        where
            A: FnMut(&dyn Point, Option<&WatchIDRef>) -> ControlFlow<()>,
        {
            fn on_point(
                &mut self,
                point: &dyn Point,
                watch: Option<&WatchIDRef>,
            ) -> ControlFlow<()> {
                (self.0)(point, watch)
            }
        }

        impl<F, R: Try<Output = (), Residual = E>, E: std::fmt::Debug> PointHandler for F
        where
            F: FnMut(&dyn Point, Option<&WatchIDRef>) -> R + 'static,
        {
            fn on_point(
                &mut self,
                point: &dyn Point,
                watch: Option<&WatchIDRef>,
            ) -> ControlFlow<()> {
                match (self)(point, watch).branch() {
                    ControlFlow::Continue(_) => ControlFlow::Continue(()),
                    ControlFlow::Break(e) => {
                        tracing::error!(?e, "break");
                        ControlFlow::Break(())
                    }
                }
            }
        }

        impl<F, S, R: Try<Output = (), Residual = E>, E> PointHandler for (F, S)
        where
            F: FnMut(&dyn Point, Option<&WatchIDRef>) -> R + 'static,
            S: FnMut(&LkQuery, StopReason, u64, u64) + 'static,
        {
            fn on_point(
                &mut self,
                point: &dyn Point,
                watch: Option<&WatchIDRef>,
            ) -> ControlFlow<()> {
                match (self.0)(point, watch).branch() {
                    ControlFlow::Continue(_) => ControlFlow::Continue(()),
                    ControlFlow::Break(_e) => ControlFlow::Break(()),
                }
            }
            fn on_close(&mut self, query: &LkQuery, reason: StopReason, total: u64, writen: u64) {
                (self.1)(query, reason, total, writen)
            }
        }
    }
}

/// common 'opinionated' functions and protocols in linkspace
pub mod commons;
pub mod experimental;

mod argument_traits;
#[cfg(any(feature = "pyo3", feature = "wasm"))]
pub mod ffi;
pub use consts::{PRIVATE, PUBLIC};
pub mod consts {
    pub use linkspace_core::consts::point_consts::*;
    pub use linkspace_core::consts::{EXCHANGE_DOMAIN, PRIVATE, PUBLIC, TEST_GROUP, test_key};
}

/// misc functions & tools - less stable
pub mod misc {
    pub use linkspace_core::point::FieldEnum;
    pub use linkspace_core::point::RecvPoint;
    pub use linkspace_core::point::point::DEFAULT_ROUTING_BITS;
    pub use linkspace_core::point::point::cmp::PointCmp;
    pub use linkspace_core::point::reroute::{MutXHeaderPoint, SharePointArc};
    pub use linkspace_core::point::space_order::SpaceDBEntry;
    pub use linkspace_core::point::{calc_free_space, deserialize::read_point_len};
    pub use misc_utils as utils;

    pub use linkspace_core::stamp_fmt::START_STAMP;

    use linkspace_core::prelude::B64;

    /// Blake3 hash
    pub fn blake3(val: &[u8]) -> B64<[u8; 32]> {
        B64(*linkspace_core::crypto::blake3(val).as_bytes())
    }

    /**
    Read bytes as a [0,1) float by reading the first 52 bits.
    Panics if fewer than 8 bytes are supplied
    Can be used to produces a shared 'random' value by using Point::hash or [blake3] as input.
     */
    pub fn bytes2uniform(val: &[u8]) -> f64 {
        let rand_u64 = u64::from_be_bytes(
            val[..8]
                .try_into()
                .expect("bytes2uniform requires at least 8 bytes"),
        );
        let f64_exponent_bits: u64 = 1023u64 << 52;
        // Generate a value in the range [1, 2)
        let value1_2 = f64::from_bits((rand_u64 >> (64 - 52)) | f64_exponent_bits);
        value1_2 - 1.0
    }

    pub static MARK_SVG: &str = include_str!("./mark.svg");
}

pub use abe::lka_eval;
/** ascii byte expression utilities

ABE is a byte templating language.
See guide#ABE for more info.
 **/
pub mod abe {
    use super::*;
    use linkspace_core::abe::abtxt::as_abtxt;
    pub use lkp::repr::DEFAULT_POINT_EXPR;

    /**
    Evaluate an expression to bytes.

    Print a list of functions by using `lka_eval("[help]")`

    Optionally add a `point` in the scope.
    See [lka_eval_strict] that is less strict on its input

    ```
    # use linkspace::{*,prelude::*,abe::*};
    # fn main() -> LkResult{
    assert_eq!( b"abc" as &[u8]    , lka_eval( "abc" ,())?, );
    assert_eq!( &[0u8,1,255] as &[u8], lka_eval( r#"\0\x01\xff"# ,())?,);

    // calling functions such as 'u8'
    assert_eq!( b"abc" as &[u8]    , lka_eval( "ab[u8:99]" ,())?, );

    assert_eq!(
               b"The '' function returns its first argument" as &[u8],
       lka_eval( "The '' function returns its first [:argument]", ())?
    );

    assert_eq!(
               b"Bytes are joined" as &[u8],
       lka_eval( "Bytes[: are][: joined]" , ())?
    );

    let result = lka_eval( "Nest expressions [u8:65] == [u8:[:65]] == \x41" , ())?;
    assert_eq!(result,   b"Nest expressions A == A == A");

    let result = lka_eval( "Use result as first argument with '/' [u8:65] == [:65/u8] == \x41" , ())?;
    assert_eq!(result,   b"Use result as first argument with '/' A == A == A");


    let result = lka_eval( "You can provide an argv [0] [1]" , &[b"like" as &[u8], b"this"])?;
    assert_eq!(result,   b"You can provide an argv like this");

    let lp : PointBox = lk_linkpoint(&[], b"mydomain", &[])?;
    let point: &dyn Point = &lp;

    assert_eq!( lka_eval( "[hash]" , point)?,&*point.hash());
    let by_arg   = lka_eval( "[hash:str]", point)?;
    let by_apply = lka_eval( "[hash/?b]",  point)?;
    let as_field = point.hash().b64().into_bytes();
    assert_eq!( by_arg, by_apply);
    assert_eq!( by_arg, as_field);

    // or provide both at once with (point,&[b"argv"])

    // escaped characters
    assert_eq!( lka_eval( r#"\n\t\:\/\\\[\]"# ,())?,  &[b'\n',b'\t',b':',b'/',b'\\',b'[',b']'] );

    # Ok(())
    # }
    ```

    **/
    pub fn lka_eval(expr: &str, scope: &dyn LkEnv) -> LkResult<Vec<u8>> {
        _lka_eval(expr, false, scope)
    }

    /**
    lka_eval and String::from_utf8
    **/
    pub fn lka_eval2str(expr: &str, scope: &dyn LkEnv) -> LkResult<String> {
        Ok(String::from_utf8(lka_eval(expr, scope)?)?)
    }

    /**
    Same as lka_eval but accepts bytes outside the range 0x20..0xfe as-is.
    useful for templating with newlines and utf bytes.

    This distinction exists because UTF has a bunch of characters that can hide a surprise -
        lka_eval input and lka_encode output is only every ascii.
    ```
    # use linkspace::{*,prelude::*,abe::*};
    # fn main() -> LkResult{
    assert_eq!( "abc 🔗🔗".as_bytes() as &[u8], &lka_eval_strict( "abc 🔗🔗" ,())?, );
    assert_eq!( "\0\0\0\0\0\0\0\0\0\0\0\0🔗 4036990103".as_bytes() as &[u8], &lka_eval_strict( "[a:🔗] [:🔗/?u]",())?, );
    # Ok(())}
    ```
    **/
    pub fn lka_eval_strict(expr: &str, scope: &dyn LkEnv) -> LkResult<Vec<u8>> {
        _lka_eval(expr, true, scope)
    }
    #[doc(hidden)]
    pub fn _lka_eval(expr: &str, strict: bool, scope: &dyn LkEnv) -> LkResult<Vec<u8>> {
        Ok(experimental::lka_eval_list(expr, strict, scope)?.concat())
    }
    /**
    encode bytes as an abe that evaluate back to bytes.

    accepts a list of func to try and encode.
    This function can also be used as the evaluator '[/?:..]'.
    ```
    # use linkspace::{*,abe::*};
    # fn main() -> LkResult{
    let bytes= lka_eval("[u32:8]",())?;
    assert_eq!(bytes,&[0,0,0,8]);
    assert_eq!(lka_encode(&bytes,"/:", &()), r#"\0\0\0\x08"#);
    assert_eq!(lka_encode(&bytes,"u32", &()), "[u32:8]");


    // the options are a list of '/' separated functions
    // In this example 'u32' won't fit, LNS '#' lookup will succeed, if not the encoding would be base64

    let public_grp = PUBLIC;
    assert_eq!(lka_encode(&*public_grp,"u32/#/b", &()), "[#:pub]");

    // We can get meta - encode is also available as a scope during lka_eval

    // As the '?' function - with the tail argument being a single reverse option
    assert_eq!(lka_eval(r#"[?:\0\0\0[u8:8]:u32]"#,())?,b"[u32:8]");
    assert_eq!(lka_eval(r#"[:\0\0\0[u8:8]/?:u32]"#,())?,b"[u32:8]");

    // Or as the '?' macro
    assert_eq!(lka_eval(r#"[/?:\0\0\0[u8:8]/u32/#/b]"#,())?,b"[u32:8]");

    # Ok(())
    # }
    ```
    encode doesn't error, it falls back on plain abtxt.
    this variant also swallows bad options, see [lk_try_encode] to avoid doing so.
    **/
    pub fn lka_encode(bytes: impl AsRef<[u8]>, options: &str, scope: &dyn LkEnv) -> String {
        let bytes = bytes.as_ref();
        try_encode_impl(bytes, options, true, scope)
            .unwrap_or_else(|_v| as_abtxt(bytes).to_string())
    }
    /// encode as utf8 and escape only invalid byte sequences and '[' ']'.
    pub fn lka_encode_utf8(bytes: &[u8]) -> String {
        linkspace_core::abe::abtxt::escape_expressions_loose(bytes).to_string()
    }
    /** [lka_encode] with Err on:
    - wrong options
    - no result ( use a /: to fallback to abtxt)
    - if !ignore_encoder_err on any encoder error
    **/
    pub fn lka_try_encode(
        bytes: impl AsRef<[u8]>,
        options: &str,
        ignore_encoder_err: bool,
        scope: &dyn LkEnv,
    ) -> LkResult<String> {
        let bytes = bytes.as_ref();
        try_encode_impl(bytes, options, ignore_encoder_err, scope)
    }
    fn try_encode_impl(
        bytes: &[u8],
        options: &str,
        ignore_encoder_err: bool,
        scope: &dyn LkEnv,
    ) -> LkResult<String> {
        Ok(linkspace_core::abe::eval::encode(
            &scope.as_scope().as_dyn(),
            bytes,
            options,
            ignore_encoder_err,
        )?)
    }
    pub fn lka_space_expr(expr: &str, scope: &dyn LkEnv) -> LkResult<Space> {
        let qexpr: linkspace_core::space_expr::SpaceExpr = expr.parse()?;
        qexpr.eval(scope.as_scope().as_dyn())
    }
}

// Allow for interop when using the linkspace_core crate
#[doc(hidden)]
pub mod interop;

/// build a custom scope for ABE for use in [varscope]
pub mod work_env {

    use core::fmt;
    use std::sync::Arc;

    use either::Either;
    use linkspace_commons_internal::{argon2_identity::eval::AdhocKey, rt_scope::RtScopeSet};
    use linkspace_core::abe::eval::{AsScope, AsScopeSet, Scope, ScopeSet};
    use linkspace_core::abe::scope::ArgV;
    use linkspace_core::eval::thread_local_scope::ScopeLocalLkEnv;
    use linkspace_core::eval::{CORE_SCOPE, CoreScope};
    use linkspace_core::point::{Domain, GroupID, LkIdentity, PointScope, PubKey};
    use linkspace_core::prelude::Point;

    #[derive(Copy, Clone, Default, Debug)]
    #[repr(C)]
    /// Data set by user for custom scopes
    pub struct LkEnvData<'o> {
        /// Set the point in scope - sets scopes like \[hash:str\], \[group\], \[domain:str]\
        pub point: Option<&'o dyn Point>,
        /// Add the argv scope e.g. "\[0\] and \[1\]"
        pub argv: Option<&'o [&'o [u8]]>,

        /// set the working domain, usable as the AB expr [wd]
        pub domain: Option<&'o Domain>,
        /// set the working group, usable as the AB expr [wg]
        pub group: Option<&'o GroupID>,
        /// set the working pubkey, usable as the AB expr [wpubkey] - this is not the same as key:pub
        pub pubkey: Option<&'o PubKey>,

        /// set the default signing key - this is unrelated to the pubkey
        pub key: Option<&'o LkIdentity>,

        #[cfg(any(feature = "lmdb", feature = "inmem"))]
        pub lks: Option<&'o super::LkSystem>,
    }

    #[derive(Clone, Debug)]
    pub struct LkEnvDataSync {
        pub point: Option<PointArc>,
        pub argv: Option<Arc<Vec<Vec<u8>>>>,
        pub domain: Option<Domain>,
        pub group: Option<GroupID>,
        pub pubkey: Option<PubKey>,
        pub key: Option<LkIdentity>,
        #[cfg(any(feature = "lmdb", feature = "inmem"))]
        pub lksi: Option<LksInfo>,
    }

    use crate::prelude::*;
    use crate::system::{LksInfo, lks_info};
    use std::{
        cell::{Cell, LazyCell},
        ptr::NonNull,
    };

    use linkspace_core::thread_local as tl;

    /// Will panic if an invalid LKE_DOMAIN expr is set - otherwise defaults to ""
    pub fn lke_domain() -> Domain {
        tl::get_domain(LkaScope::default().as_dyn())
    }
    pub fn lke_set_domain(domain: Domain) {
        let _ = tl::set_domain(Some(domain), true);
    }

    /// Will panic if an invalid LKE_GROUP expr is set - otherwise defaults to "[#:pub]"
    pub fn lke_group() -> GroupID {
        tl::get_group(LkaScope::default().as_dyn())
    }
    pub fn lke_set_group(group: GroupID) {
        let _ = tl::set_group(Some(group), true);
    }
    /// (Not to be confused with lka_eval("[keypub]") ) - Will panic if an invalid LK_PUBKEY expr is set - otherwise defaults to "[@:0]"
    pub fn lke_pubkey() -> PubKey {
        tl::get_pubkey(LkaScope::default().as_dyn())
    }
    pub fn lke_key() -> Option<LkIdentity> {
        tl::get_key()
    }
    pub fn lke_set_key(id: LkIdentity) {
        let _ = tl::set_key(Some(id), true);
    }
    /// The open signing key's pubkey, i.e. [get_key].pubkey or lka_eval("[keypub]")
    pub fn lke_keypub() -> Option<PubKey> {
        tl::get_keypub()
    }

    // we must not leak the lifetimes from the thread locals
    pub fn lke_with<A>(cb: impl FnOnce(crate::work_env::LkEnvData) -> A) -> A {
        let point: Option<&dyn Point> = match unsafe { &*crate::work_env::ENV_POINT.as_ptr() } {
            None => None,
            Some(Either::Left(l)) => Some(l),
            Some(Either::Right(l)) => Some(unsafe { l.as_ref() }),
        };
        let argv: Option<&[&[u8]]> = match unsafe { &*crate::work_env::ENV_ARGV.as_ptr() } {
            None => None,
            Some((bytes, _)) => Some(bytes),
        };
        use linkspace_system::thread_local as tls;
        let lks = tls::opt_get_system();
        cb(crate::work_env::LkEnvData {
            point,
            argv,
            group: unsafe { &*tl::GROUP.as_ptr() }.as_ref(),
            pubkey: unsafe { &*tl::PUBKEY.as_ptr() }.as_ref(),
            domain: unsafe { &*tl::DOMAIN.as_ptr() }.as_ref(),
            lks: lks.as_ref().map(crate::LkSystem::from_impl),
            key: unsafe { &*tl::SIGNING_KEY.as_ptr() }.as_ref(),
        })
    }

    pub fn lka_set_keyval(key: &[&[u8]], value: &[u8]) -> crate::LkResult<Vec<u8>> {
        linkspace_core::thread_local::set_keyval_map(key, value)
    }
    #[allow(clippy::type_complexity)]
    pub fn lka_set_callback(cb: Box<dyn Fn(&[&[u8]], bool) -> crate::LkResult<Option<Vec<u8>>>>) {
        linkspace_core::thread_local::set_shared_cb(cb)
    }
    #[thread_local]
    pub static DUMMY_RT: LazyCell<crate::LkSystem> =
        LazyCell::new(|| crate::system::lks_open_inmem("_DUMMY_"));

    pub fn lke_set(scope: &dyn LkEnv) {
        let mut ud = crate::work_env::LkEnvData::default();
        scope.userdata(&mut ud);
        if let Some(g) = ud.group {
            let _ = tl::set_group(Some(*g), true);
        }
        if let Some(d) = ud.domain {
            let _ = tl::set_domain(Some(*d), true);
        }
        if let Some(p) = ud.pubkey {
            let _ = tl::set_pubkey(Some(*p), true);
        }
        if let Some(p) = ud.key {
            let _ = tl::set_key(Some(p.clone()), true);
        }
        if let Some(d) = ud.point {
            ENV_POINT.set(Some(Either::Left(d.as_arc())));
        }
        if let Some(rt) = ud.lks {
            unsafe { set_system(rt.clone().into()) };
        }
        if let Some(d) = ud.argv {
            let argv_bytes = d.concat().into_boxed_slice();
            let mut argv: [&[u8]; 10] = [&[]; 10];
            let mut slice: &[u8] = &argv_bytes;
            for (bytes, dest) in d.iter().zip(argv.iter_mut()) {
                let b: &[u8] = &slice[..bytes.len()];
                *dest = unsafe { std::slice::from_ptr_range(b.as_ptr_range()) };
                slice = &slice[bytes.len()..]
            }
            ENV_ARGV.set(Some((argv, Some(argv_bytes))));
        }
    }
    /// A Sync lke struct.
    pub fn lke_sync() -> LkEnvDataSync {
        lke_with(|e| LkEnvDataSync {
            point: e.point.map(|p| p.as_arc()),
            argv: e
                .argv
                .map(|o| Arc::new(o.iter().map(|o| o.to_vec()).collect())),
            domain: e.domain.copied(),
            group: e.group.copied(),
            pubkey: e.pubkey.copied(),
            key: e.key.cloned(),
            lksi: e.lks.map(|i| lks_info(Some(i)).unwrap()),
        })
    }

    #[thread_local]
    pub(crate) static ENV_POINT: Cell<Option<Either<PointArc, NonNull<dyn Point>>>> =
        Cell::new(None);

    #[thread_local]
    pub(crate) static ENV_ARGV: Cell<Option<([&'static [u8]; 10], Option<Box<[u8]>>)>> =
        Cell::new(None);

    pub fn lke_point_set(point: &dyn Point) {
        ENV_POINT.set(Some(Either::Left(point.as_arc())));
    }
    pub fn lke_point_do<A>(point: &dyn Point, fnc: impl FnOnce() -> A) -> A {
        let prev = ENV_POINT
            .replace(NonNull::new(point as *const dyn Point as *mut dyn Point).map(Either::Right));
        let r = fnc();
        ENV_POINT.set(prev);
        r
    }
    pub fn lke_point() -> Option<PointArc> {
        match ENV_POINT.take() {
            None => None,
            Some(Either::Left(l)) => {
                ENV_POINT.set(Some(Either::Left(l.clone())));
                Some(l)
            }
            Some(Either::Right(r)) => {
                let arc = unsafe { r.as_ref().as_arc() };
                ENV_POINT.set(Some(Either::Left(arc.clone())));
                Some(arc)
            }
        }
    }

    /**
    Temporarily hoist the user scope (lks,group, domain, and pubkey) into the threadlocals.
    */
    pub fn lke_do<A>(scope: &dyn LkEnv, fnc: impl FnOnce() -> A) -> A {
        let mut ud = crate::work_env::LkEnvData::default();
        scope.userdata(&mut ud);

        let prev_group = if let Some(g) = ud.group {
            tl::set_group(Some(*g), true).unwrap()
        } else {
            None
        };
        let prev_domain = if let Some(d) = ud.domain {
            tl::set_domain(Some(*d), true).unwrap()
        } else {
            None
        };
        let prev_pubkey = if let Some(k) = ud.pubkey {
            tl::set_pubkey(Some(*k), true).unwrap()
        } else {
            None
        };
        let prev_key = if let Some(k) = ud.key {
            tl::set_key(Some(k.clone()), true).unwrap()
        } else {
            None
        };
        let prev_argv = if let Some(slices) = ud.argv {
            let mut argv = [&[] as &[u8]; 10];
            for (bytes, dest) in slices.iter().zip(argv.iter_mut()) {
                *dest = unsafe { std::slice::from_ptr_range(bytes.as_ptr_range()) };
            }
            ENV_ARGV.replace(Some((argv, None)))
        } else {
            None
        };

        let prev_point = if let Some(point) = ud.point {
            ENV_POINT.replace(
                NonNull::new(point as *const dyn Point as *mut dyn Point).map(Either::Right),
            )
        } else {
            None
        };
        let result = if let Some(lks) = ud.lks {
            crate::system::lks_chsys(lks, fnc)
        } else {
            fnc()
        };
        if prev_group.is_some() {
            let _ = tl::set_group(prev_group, true);
        }
        if prev_domain.is_some() {
            let _ = tl::set_domain(prev_domain, true);
        }
        if prev_pubkey.is_some() {
            let _ = tl::set_pubkey(prev_pubkey, true);
        }
        if prev_key.is_some() {
            let _ = tl::set_key(prev_key, true);
        }
        if prev_point.is_some() {
            ENV_POINT.set(prev_point);
        }
        if prev_argv.is_some() {
            ENV_ARGV.set(prev_argv);
        }
        result
    }

    impl Default for LkaScope<'_> {
        fn default() -> Self {
            LkEnvData::default().into()
        }
    }
    impl fmt::Debug for LkaScope<'_> {
        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
            f.debug_tuple("LkScope").field(&"_").finish()
        }
    }

    #[derive(Clone)]
    /// Functions/Macros used in abe evaluation.
    pub struct LkaScope<'o>(pub(crate) InlineScope<'o>);
    #[derive(Clone)]
    pub(crate) enum InlineScope<'o> {
        Std(StdScope<'o>),
        Core,
        Dyn(&'o dyn Scope),
    }
    type StdScope<'o> = ScopeSet<LkEnvData<'o>>;

    impl<'o> AsScopeSet for LkEnvData<'o> {
        type Scope = (
            (Option<ScopeSet<PointScope<'o>>>, Option<AsScope<ArgV<'o>>>),
            (
                Option<AsScope<ScopeLocalLkEnv<'o>>>,
                CoreScope,
                AsScope<AdhocKey>,
            ),
            ScopeSet<RtScopeSet<SystemOrThreadLocal<'o>>>,
        );
        fn as_scope_set(&self) -> Self::Scope {
            let mut workenv = None;
            if self.domain.is_some()
                || self.group.is_some()
                || self.pubkey.is_some()
                || self.key.is_some()
            {
                workenv = Some(AsScope(ScopeLocalLkEnv {
                    domain: self.domain,
                    group: self.group,
                    pubkey: self.pubkey,
                    key: self.key,
                }));
            }
            let point_scope = self
                .point
                .or_else(|| match unsafe { &*crate::work_env::ENV_POINT.as_ptr() } {
                    None => None,
                    Some(Either::Left(l)) => Some(l),
                    Some(Either::Right(l)) => Some(unsafe { l.as_ref() }),
                })
                .map(linkspace_core::point::point_scope);
            let argv_scope = self
                .argv
                .or_else(|| match unsafe { &*crate::work_env::ENV_ARGV.as_ptr() } {
                    None => None,
                    Some((bytes, _)) => Some(bytes),
                })
                .map(|bytes| AsScope(ArgV(bytes)));
            let lks = ScopeSet(RtScopeSet(SystemOrThreadLocal(self.lks.map(|o| &o.0))));
            (
                (point_scope, argv_scope),
                (workenv, CORE_SCOPE, AsScope(AdhocKey)),
                lks,
            )
        }
    }

    // This intermediate  struct exists to allow expanding UserData with scope instances
    // FIXME: this logic is backwards to support the cli.
    impl<'o> LkaScope<'o> {
        pub fn core() -> LkaScope<'static> {
            LkaScope(InlineScope::Core)
        }
        pub(crate) fn as_dyn(&self) -> &(dyn Scope + 'o) {
            match &self.0 {
                InlineScope::Std(scope) => scope,
                InlineScope::Dyn(scope) => scope,
                InlineScope::Core => &CORE_SCOPE,
            }
        }
        #[doc(hidden)]
        pub fn from_dyn(sdyn: &'o dyn Scope) -> Self {
            LkaScope(InlineScope::Dyn(sdyn))
        }
    }
    use linkspace_system::thread_local::{SystemOrThreadLocal, set_system};
}