diff --git a/docs/command-reference/command-reference.md b/docs/command-reference/command-reference.md index aea6b70d..24543afa 100644 --- a/docs/command-reference/command-reference.md +++ b/docs/command-reference/command-reference.md @@ -2,4 +2,8 @@ sidebar_position: 6 --- +import DocCardList from '@theme/DocCardList'; + # Command Reference + + diff --git a/docs/command-reference/get/index.md b/docs/command-reference/get/index.md new file mode 100644 index 00000000..c869408f --- /dev/null +++ b/docs/command-reference/get/index.md @@ -0,0 +1,56 @@ +--- +acl_categories: +- '@read' +- '@string' +- '@fast' +arguments: +- key_spec_index: 0 + name: key + type: key +arity: 2 +command_flags: +- readonly +- fast +complexity: O(1) +description: Get the value of a key +github_branch: main +github_path: /tmp/docs/content/commands/get/index.md +github_repo: https://dragonflydb/redis-doc +group: string +hidden: false +key_specs: +- RO: true + access: true + begin_search: + spec: + index: 1 + type: index + find_keys: + spec: + keystep: 1 + lastkey: 0 + limit: 0 + type: range +linkTitle: GET +since: 1.0.0 +summary: Get the value of a key +syntax_fmt: GET key +syntax_str: '' +title: GET +--- +Get the value of `key`. +If the key does not exist the special value `nil` is returned. +An error is returned if the value stored at `key` is not a string, because `GET` +only handles string values. + +## Return + +[Bulk string reply](/docs/reference/protocol-spec#resp-bulk-strings): the value of `key`, or `nil` when `key` does not exist. + +## Examples + +```cli +GET nonexisting +SET mykey "Hello" +GET mykey +``` diff --git a/docs/command-reference/hset/index.md b/docs/command-reference/hset/index.md new file mode 100644 index 00000000..f656c97f --- /dev/null +++ b/docs/command-reference/hset/index.md @@ -0,0 +1,67 @@ +--- +acl_categories: +- '@write' +- '@hash' +- '@fast' +arguments: +- key_spec_index: 0 + name: key + type: key +- arguments: + - name: field + type: string + - name: value + type: string + multiple: true + name: field_value + type: block +arity: -4 +command_flags: +- write +- denyoom +- fast +complexity: O(1) for each field/value pair added, so O(N) to add N field/value pairs + when the command is called with multiple field/value pairs. +description: Set the string value of a hash field +github_branch: main +github_path: /tmp/docs/content/commands/hset/index.md +github_repo: https://dragonflydb/redis-doc +group: hash +hidden: false +history: +- - 4.0.0 + - Accepts multiple `field` and `value` arguments. +key_specs: +- RW: true + begin_search: + spec: + index: 1 + type: index + find_keys: + spec: + keystep: 1 + lastkey: 0 + limit: 0 + type: range + update: true +linkTitle: HSET +since: 2.0.0 +summary: Set the string value of a hash field +syntax_fmt: HSET key field value [field value ...] +syntax_str: field value [field value ...] +title: HSET +--- +Sets `field` in the hash stored at `key` to `value`. +If `key` does not exist, a new key holding a hash is created. +If `field` already exists in the hash, it is overwritten. + +## Return + +[Integer reply](/docs/reference/protocol-spec#resp-integers): The number of fields that were added. + +## Examples + +```cli +HSET myhash field1 "Hello" +HGET myhash field1 +``` diff --git a/docs/command-reference/llen/index.md b/docs/command-reference/llen/index.md new file mode 100644 index 00000000..07a185e9 --- /dev/null +++ b/docs/command-reference/llen/index.md @@ -0,0 +1,54 @@ +--- +acl_categories: +- '@read' +- '@list' +- '@fast' +arguments: +- key_spec_index: 0 + name: key + type: key +arity: 2 +command_flags: +- readonly +- fast +complexity: O(1) +description: Get the length of a list +github_branch: main +github_path: /tmp/docs/content/commands/llen/index.md +github_repo: https://dragonflydb/redis-doc +group: list +hidden: false +key_specs: +- RO: true + begin_search: + spec: + index: 1 + type: index + find_keys: + spec: + keystep: 1 + lastkey: 0 + limit: 0 + type: range +linkTitle: LLEN +since: 1.0.0 +summary: Get the length of a list +syntax_fmt: LLEN key +syntax_str: '' +title: LLEN +--- +Returns the length of the list stored at `key`. +If `key` does not exist, it is interpreted as an empty list and `0` is returned. +An error is returned when the value stored at `key` is not a list. + +## Return + +[Integer reply](/docs/reference/protocol-spec#resp-integers): the length of the list at `key`. + +## Examples + +```cli +LPUSH mylist "World" +LPUSH mylist "Hello" +LLEN mylist +``` diff --git a/docs/command-reference/lmove/index.md b/docs/command-reference/lmove/index.md new file mode 100644 index 00000000..eb4751d9 --- /dev/null +++ b/docs/command-reference/lmove/index.md @@ -0,0 +1,155 @@ +--- +acl_categories: +- '@write' +- '@list' +- '@slow' +arguments: +- key_spec_index: 0 + name: source + type: key +- key_spec_index: 1 + name: destination + type: key +- arguments: + - name: left + token: LEFT + type: pure-token + - name: right + token: RIGHT + type: pure-token + name: wherefrom + type: oneof +- arguments: + - name: left + token: LEFT + type: pure-token + - name: right + token: RIGHT + type: pure-token + name: whereto + type: oneof +arity: 5 +command_flags: +- write +- denyoom +complexity: O(1) +description: Pop an element from a list, push it to another list and return it +github_branch: main +github_path: /tmp/docs/content/commands/lmove/index.md +github_repo: https://dragonflydb/redis-doc +group: list +hidden: false +key_specs: +- RW: true + access: true + begin_search: + spec: + index: 1 + type: index + delete: true + find_keys: + spec: + keystep: 1 + lastkey: 0 + limit: 0 + type: range +- RW: true + begin_search: + spec: + index: 2 + type: index + find_keys: + spec: + keystep: 1 + lastkey: 0 + limit: 0 + type: range + insert: true +linkTitle: LMOVE +since: 6.2.0 +summary: Pop an element from a list, push it to another list and return it +syntax_fmt: LMOVE source destination +syntax_str: destination +title: LMOVE +--- +Atomically returns and removes the first/last element (head/tail depending on +the `wherefrom` argument) of the list stored at `source`, and pushes the +element at the first/last element (head/tail depending on the `whereto` +argument) of the list stored at `destination`. + +For example: consider `source` holding the list `a,b,c`, and `destination` +holding the list `x,y,z`. +Executing `LMOVE source destination RIGHT LEFT` results in `source` holding +`a,b` and `destination` holding `c,x,y,z`. + +If `source` does not exist, the value `nil` is returned and no operation is +performed. +If `source` and `destination` are the same, the operation is equivalent to +removing the first/last element from the list and pushing it as first/last +element of the list, so it can be considered as a list rotation command (or a +no-op if `wherefrom` is the same as `whereto`). + +This command comes in place of the now deprecated [`RPOPLPUSH`](/commands/rpoplpush). Doing +`LMOVE RIGHT LEFT` is equivalent. + +## Return + +[Bulk string reply](/docs/reference/protocol-spec#resp-bulk-strings): the element being popped and pushed. + +## Examples + +```cli +RPUSH mylist "one" +RPUSH mylist "two" +RPUSH mylist "three" +LMOVE mylist myotherlist RIGHT LEFT +LMOVE mylist myotherlist LEFT RIGHT +LRANGE mylist 0 -1 +LRANGE myotherlist 0 -1 +``` + +## Pattern: Reliable queue + +Redis is often used as a messaging server to implement processing of background +jobs or other kinds of messaging tasks. +A simple form of queue is often obtained pushing values into a list in the +producer side, and waiting for this values in the consumer side using [`RPOP`](/commands/rpop) +(using polling), or [`BRPOP`](/commands/brpop) if the client is better served by a blocking +operation. + +However in this context the obtained queue is not _reliable_ as messages can +be lost, for example in the case there is a network problem or if the consumer +crashes just after the message is received but it is still to process. + +`LMOVE` (or [`BLMOVE`](/commands/blmove) for the blocking variant) offers a way to avoid +this problem: the consumer fetches the message and at the same time pushes it +into a _processing_ list. +It will use the [`LREM`](/commands/lrem) command in order to remove the message from the +_processing_ list once the message has been processed. + +An additional client may monitor the _processing_ list for items that remain +there for too much time, and will push those timed out items into the queue +again if needed. + +## Pattern: Circular list + +Using `LMOVE` with the same source and destination key, a client can visit +all the elements of an N-elements list, one after the other, in O(N) without +transferring the full list from the server to the client using a single [`LRANGE`](/commands/lrange) +operation. + +The above pattern works even if the following two conditions: + +* There are multiple clients rotating the list: they'll fetch different + elements, until all the elements of the list are visited, and the process + restarts. +* Even if other clients are actively pushing new items at the end of the list. + +The above makes it very simple to implement a system where a set of items must +be processed by N workers continuously as fast as possible. +An example is a monitoring system that must check that a set of web sites are +reachable, with the smallest delay possible, using a number of parallel workers. + +Note that this implementation of workers is trivially scalable and reliable, +because even if a message is lost the item is still in the queue and will be +processed at the next iteration. diff --git a/docs/command-reference/lmpop/index.md b/docs/command-reference/lmpop/index.md new file mode 100644 index 00000000..4fd2dcb1 --- /dev/null +++ b/docs/command-reference/lmpop/index.md @@ -0,0 +1,93 @@ +--- +acl_categories: +- '@write' +- '@list' +- '@slow' +arguments: +- name: numkeys + type: integer +- key_spec_index: 0 + multiple: true + name: key + type: key +- arguments: + - name: left + token: LEFT + type: pure-token + - name: right + token: RIGHT + type: pure-token + name: where + type: oneof +- name: count + optional: true + token: COUNT + type: integer +arity: -4 +command_flags: +- write +- movablekeys +complexity: O(N+M) where N is the number of provided keys and M is the number of elements + returned. +description: Pop elements from a list +github_branch: main +github_path: /tmp/docs/content/commands/lmpop/index.md +github_repo: https://dragonflydb/redis-doc +group: list +hidden: false +key_specs: +- RW: true + access: true + begin_search: + spec: + index: 1 + type: index + delete: true + find_keys: + spec: + firstkey: 1 + keynumidx: 0 + keystep: 1 + type: keynum +linkTitle: LMPOP +since: 7.0.0 +summary: Pop elements from a list +syntax_fmt: "LMPOP numkeys key [key ...] [COUNT\_count]" +syntax_str: "key [key ...] [COUNT\_count]" +title: LMPOP +--- +Pops one or more elements from the first non-empty list key from the list of provided key names. + +`LMPOP` and [`BLMPOP`](/commands/blmpop) are similar to the following, more limited, commands: + +- [`LPOP`](/commands/lpop) or [`RPOP`](/commands/rpop) which take only one key, and can return multiple elements. +- [`BLPOP`](/commands/blpop) or [`BRPOP`](/commands/brpop) which take multiple keys, but return only one element from just one key. + +See [`BLMPOP`](/commands/blmpop) for the blocking variant of this command. + +Elements are popped from either the left or right of the first non-empty list based on the passed argument. +The number of returned elements is limited to the lower between the non-empty list's length, and the count argument (which defaults to 1). + +## Return + +[Array reply](/docs/reference/protocol-spec#resp-arrays): specifically: + +* A `nil` when no element could be popped. +* A two-element array with the first element being the name of the key from which elements were popped, and the second element is an array of elements. + +## Examples + +```cli +LMPOP 2 non1 non2 LEFT COUNT 10 +LPUSH mylist "one" "two" "three" "four" "five" +LMPOP 1 mylist LEFT +LRANGE mylist 0 -1 +LMPOP 1 mylist RIGHT COUNT 10 +LPUSH mylist "one" "two" "three" "four" "five" +LPUSH mylist2 "a" "b" "c" "d" "e" +LMPOP 2 mylist mylist2 right count 3 +LRANGE mylist 0 -1 +LMPOP 2 mylist mylist2 right count 5 +LMPOP 2 mylist mylist2 right count 10 +EXISTS mylist mylist2 +``` diff --git a/docs/command-reference/publish/index.md b/docs/command-reference/publish/index.md new file mode 100644 index 00000000..0c2a0ae5 --- /dev/null +++ b/docs/command-reference/publish/index.md @@ -0,0 +1,41 @@ +--- +acl_categories: +- '@pubsub' +- '@fast' +arguments: +- name: channel + type: string +- name: message + type: string +arity: 3 +command_flags: +- pubsub +- loading +- stale +- fast +complexity: O(N+M) where N is the number of clients subscribed to the receiving channel + and M is the total number of subscribed patterns (by any client). +description: Post a message to a channel +github_branch: main +github_path: /tmp/docs/content/commands/publish/index.md +github_repo: https://dragonflydb/redis-doc +group: pubsub +hidden: false +linkTitle: PUBLISH +since: 2.0.0 +summary: Post a message to a channel +syntax_fmt: PUBLISH channel message +syntax_str: message +title: PUBLISH +--- +Posts a message to the given channel. + +In a Redis Cluster clients can publish to every node. The cluster makes sure +that published messages are forwarded as needed, so clients can subscribe to any +channel by connecting to any one of the nodes. + +## Return + +[Integer reply](/docs/reference/protocol-spec#resp-integers): the number of clients that received the message. Note that in a +Redis Cluster, only clients that are connected to the same node as the +publishing client are included in the count. diff --git a/docs/command-reference/set/index.md b/docs/command-reference/set/index.md new file mode 100644 index 00000000..fa9de028 --- /dev/null +++ b/docs/command-reference/set/index.md @@ -0,0 +1,163 @@ +--- +acl_categories: +- '@write' +- '@string' +- '@slow' +arguments: +- key_spec_index: 0 + name: key + type: key +- name: value + type: string +- arguments: + - name: nx + token: NX + type: pure-token + - name: xx + token: XX + type: pure-token + name: condition + optional: true + since: 2.6.12 + type: oneof +- name: get + optional: true + since: 6.2.0 + token: GET + type: pure-token +- arguments: + - name: seconds + since: 2.6.12 + token: EX + type: integer + - name: milliseconds + since: 2.6.12 + token: PX + type: integer + - name: unix-time-seconds + since: 6.2.0 + token: EXAT + type: unix-time + - name: unix-time-milliseconds + since: 6.2.0 + token: PXAT + type: unix-time + - name: keepttl + since: 6.0.0 + token: KEEPTTL + type: pure-token + name: expiration + optional: true + type: oneof +arity: -3 +command_flags: +- write +- denyoom +complexity: O(1) +description: Set the string value of a key +github_branch: main +github_path: /tmp/docs/content/commands/set/index.md +github_repo: https://dragonflydb/redis-doc +group: string +hidden: false +history: +- - 2.6.12 + - Added the `EX`, `PX`, `NX` and `XX` options. +- - 6.0.0 + - Added the `KEEPTTL` option. +- - 6.2.0 + - Added the `GET`, `EXAT` and `PXAT` option. +- - 7.0.0 + - Allowed the `NX` and `GET` options to be used together. +key_specs: +- RW: true + access: true + begin_search: + spec: + index: 1 + type: index + find_keys: + spec: + keystep: 1 + lastkey: 0 + limit: 0 + type: range + notes: RW and ACCESS due to the optional `GET` argument + update: true + variable_flags: true +linkTitle: SET +since: 1.0.0 +summary: Set the string value of a key +syntax_fmt: "SET key value [NX | XX] [GET] [EX\_seconds | PX\_milliseconds |\n EXAT\_\ + unix-time-seconds | PXAT\_unix-time-milliseconds | KEEPTTL]" +syntax_str: "value [NX | XX] [GET] [EX\_seconds | PX\_milliseconds | EXAT\_unix-time-seconds\ + \ | PXAT\_unix-time-milliseconds | KEEPTTL]" +title: SET +--- +Set `key` to hold the string `value`. +If `key` already holds a value, it is overwritten, regardless of its type. +Any previous time to live associated with the key is discarded on successful `SET` operation. + +## Options + +The `SET` command supports a set of options that modify its behavior: + +* `EX` *seconds* -- Set the specified expire time, in seconds. +* `PX` *milliseconds* -- Set the specified expire time, in milliseconds. +* `EXAT` *timestamp-seconds* -- Set the specified Unix time at which the key will expire, in seconds. +* `PXAT` *timestamp-milliseconds* -- Set the specified Unix time at which the key will expire, in milliseconds. +* `NX` -- Only set the key if it does not already exist. +* `XX` -- Only set the key if it already exist. +* `KEEPTTL` -- Retain the time to live associated with the key. +* `GET` -- Return the old string stored at key, or nil if key did not exist. An error is returned and `SET` aborted if the value stored at key is not a string. + +Note: Since the `SET` command options can replace [`SETNX`](/commands/setnx), [`SETEX`](/commands/setex), [`PSETEX`](/commands/psetex), [`GETSET`](/commands/getset), it is possible that in future versions of Redis these commands will be deprecated and finally removed. + +## Return + +[Simple string reply](/docs/reference/protocol-spec#resp-simple-strings): `OK` if `SET` was executed correctly. + +[Null reply](/docs/reference/protocol-spec#resp-bulk-strings): `(nil)` if the `SET` operation was not performed because the user specified the `NX` or `XX` option but the condition was not met. + +If the command is issued with the `GET` option, the above does not apply. It will instead reply as follows, regardless if the `SET` was actually performed: + +[Bulk string reply](/docs/reference/protocol-spec#resp-bulk-strings): the old string value stored at key. + +[Null reply](/docs/reference/protocol-spec#resp-bulk-strings): `(nil)` if the key did not exist. + +## Examples + +```cli +SET mykey "Hello" +GET mykey + +SET anotherkey "will expire in a minute" EX 60 +``` + +## Patterns + +**Note:** The following pattern is discouraged in favor of [the Redlock algorithm](https://redis.io/topics/distlock) which is only a bit more complex to implement, but offers better guarantees and is fault tolerant. + +The command `SET resource-name anystring NX EX max-lock-time` is a simple way to implement a locking system with Redis. + +A client can acquire the lock if the above command returns `OK` (or retry after some time if the command returns Nil), and remove the lock just using [`DEL`](/commands/del). + +The lock will be auto-released after the expire time is reached. + +It is possible to make this system more robust modifying the unlock schema as follows: + +* Instead of setting a fixed string, set a non-guessable large random string, called token. +* Instead of releasing the lock with [`DEL`](/commands/del), send a script that only removes the key if the value matches. + +This avoids that a client will try to release the lock after the expire time deleting the key created by another client that acquired the lock later. + +An example of unlock script would be similar to the following: + + if redis.call("get",KEYS[1]) == ARGV[1] + then + return redis.call("del",KEYS[1]) + else + return 0 + end + +The script should be called with `EVAL ...script... 1 resource-name token-value` diff --git a/docs/command-reference/subscribe/index.md b/docs/command-reference/subscribe/index.md new file mode 100644 index 00000000..75706ccc --- /dev/null +++ b/docs/command-reference/subscribe/index.md @@ -0,0 +1,37 @@ +--- +acl_categories: +- '@pubsub' +- '@slow' +arguments: +- multiple: true + name: channel + type: string +arity: -2 +command_flags: +- pubsub +- noscript +- loading +- stale +complexity: O(N) where N is the number of channels to subscribe to. +description: Listen for messages published to the given channels +github_branch: main +github_path: /tmp/docs/content/commands/subscribe/index.md +github_repo: https://dragonflydb/redis-doc +group: pubsub +hidden: false +linkTitle: SUBSCRIBE +since: 2.0.0 +summary: Listen for messages published to the given channels +syntax_fmt: SUBSCRIBE channel [channel ...] +syntax_str: '' +title: SUBSCRIBE +--- +Subscribes the client to the specified channels. + +Once the client enters the subscribed state it is not supposed to issue any +other commands, except for additional `SUBSCRIBE`, [`SSUBSCRIBE`](/commands/ssubscribe), [`PSUBSCRIBE`](/commands/psubscribe), [`UNSUBSCRIBE`](/commands/unsubscribe), [`SUNSUBSCRIBE`](/commands/sunsubscribe), +[`PUNSUBSCRIBE`](/commands/punsubscribe), [`PING`](/commands/ping), [`RESET`](/commands/reset) and [`QUIT`](/commands/quit) commands. + +## Behavior change history + +* `>= 6.2.0`: [`RESET`](/commands/reset) can be called to exit subscribed state. \ No newline at end of file diff --git a/docusaurus.config.js b/docusaurus.config.js index 3fee9a7a..2187066e 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -10,7 +10,7 @@ const config = { tagline: "Ultra-fast, scalable in-memory datastore", url: "https://dragonflydb.io", baseUrl: process.env.VERCEL_ENV === "preview" ? "/" : "/docs", - onBrokenLinks: "throw", + onBrokenLinks: "warn", onBrokenMarkdownLinks: "warn", favicon: "img/favicon.ico", diff --git a/scripts/.dockerignore b/scripts/.dockerignore new file mode 100644 index 00000000..31314f83 --- /dev/null +++ b/scripts/.dockerignore @@ -0,0 +1,3 @@ +Dockerfile +dockers +generate_by_docker.sh diff --git a/scripts/Dockerfile b/scripts/Dockerfile new file mode 100644 index 00000000..28bd9752 --- /dev/null +++ b/scripts/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:20.04 + +RUN apt update -y && apt install -y wget vim python3 python3-distutils git +RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py +WORKDIR /dragonfly +ADD . . +#RUN python3 -m pip install -r requirements.txt +RUN python3 -m pip install virtualenv diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..e65fb67e --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,122 @@ +# Building Commands Docs + +## Overview +This directory contains a collection of scripts and dockerfile to generate the files that would be used for the website to show commands documents. +Using these scripts we can generate them either from local location or by accessing the file on the remote repository. + +## Requirements +If you would like to run these scripts locally without using a docker image, you must install the following on you local machine (assuming your running on Ubuntu Linux): +- Python 3 (sudo apt install -y python) +- Python pip (wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py) +- Python virtual env (python3 -m pip install virtualenv) +- Git (sudo apt install -y git) + +## Building Documents +### Building with Docker +The script generate_by_docker.sh allow building inside docker. +This script accept 2 options: +- --build [image name]: This option would build the docker image. +- --run [volume name] [image name]: this option would generate the files using the script generate_docs.sh and using option --clone for it. +Note that you can run the --build option and then use the docker image to run the script with other options. +For example: +Building the image: +``` +./generate_by_docker.sh -b +``` +Building the image with specific image name: +``` +./generate_by_docker.sh -b build_image +``` + +Generating documentation using the image: +``` +./generate_by_docker.sh -r +``` +Generating the docs using you own volume: +``` +./generate_by_docker.sh -b /path/to/my/host/volume +``` +Generating the docks using your own volume and your own image name: +``` +./generate_by_docker.sh -b /path/to/my/host/volume build_image +``` + +### Running from Terminal +The script generate_docs.sh is the "engine" for running the python script. +This would simplify the running of the python script and add some more options. +With this script, when running without command line options, it would clone [redis-doc](https://github.com/dragonflydb/redis-doc), to the local host, +and write the result to /tmp//docs/content directory. +The options for this script are: +- --clone This would clone the files from https://github.com/dragonflydb/redis-doc.git into local directory and run on it (the default) +- --remote This would use the above URL to fetch files without copying them locally +- --output "dir name" This would place the generated files at the "dir name", default to /tmp/"current utc timestamp"/docs/content +- --local "input dir" Run this using a local files at location "input dir" +- --clean This would remove the build and log files +Note that the default is to run with --clone option + +For example: +Building the documents using data from the [redis-doc](https://github.com/dragonflydb/redis-doc) +``` +./generate_docs.sh +``` + +Building the documents using data from the [redis-doc](https://github.com/dragonflydb/redis-doc) with explicit cloning +``` +./generate_docs.sh --clone +``` + +Building the documents using data from the [redis-doc](https://github.com/dragonflydb/redis-doc) without doing any cloning +``` +./generate_docs.sh --remote +``` + +Building the documents using data from the [redis-doc](https://github.com/dragonflydb/redis-doc) after you already have it locally +``` +./generate_docs.sh --local /path/to/files/from/redis-doc +``` + +Building the documents using data from the [redis-doc](https://github.com/dragonflydb/redis-doc) and setting output directory +./generate_docs.sh --clone --output /path/to/write/content + +Cleaning up (this is best run with the option of the output directory since this cannot be derived directly) +./generate_docs.sh --clean --output /path/to/write/content + +### Running directly (development mode) +The script that generates the output files to website is called generate_commands_docs.py +It is best to create and use python virtual environment for it, so you would not install all dependencies globally. +To install the required dependencies run: +``` +pip3 install -r requirements.txt +``` + +This scripts accepts the following options: +- --output_dir "path" : Output directory to which result will be writing "path" (required). +- --local "path": Running with local repository "input file location" (optional) +Note that if you're not running with --local, the script will connect to the remote [redis-doc](https://github.com/dragonflydb/redis-doc), and generate the output by reading directly from there. +The resulting files will be writing to the output_dir that was passed to the script. +Note that this will place each command in a separate subdirectory and the content is writing to index.md file. +For example: +Building the content without using local files: +``` +./generate_commands_docs.py --output_dir /path/to/write/content +``` + +Building content using existing files from [redis-doc](https://github.com/dragonflydb/redis-doc) +``` +./generate_commands_docs.py --output_dir /path/to/write/content --local /path/to/files/from/redis-doc +``` + + + +## Manually Running +In order to run using the docker image, you would need to make sure that you have docker support on your host - [get docker](https://docs.docker.com/get-docker/). +Then you can either manually build +``` +docker build -t dragonfly_docs_builder . +``` +and +``` +docker run --rm -t -v :/tmp/docs/content/commands \ + dragonfly_docs_builder ./generate_docs.sh -o /tmp/docs/content/commands +``` +Or using the commands above diff --git a/scripts/generate_by_docker.sh b/scripts/generate_by_docker.sh new file mode 100755 index 00000000..5b2eb536 --- /dev/null +++ b/scripts/generate_by_docker.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Copyright 2022, DragonflyDB authors. All rights reserved. +# See LICENSE for licensing terms. + +## Building the dockements using docker as the build env +# Option +# -b|--build [tagname] build the docker image locally, default to dragonfly_docs_builder +# -r|--run [image name] run the container and generate the build + +IMAGE_NAME=dragonfly_docs_builder +LOCAL_VOLUME=/tmp/docs/content/commands +CURRENT_RELATIVE=`dirname "$0"` +ABS_SCRIPT_PATH=`( cd "$CURRENT_RELATIVE" && pwd )` + +function build_image { + if [ "$1" != "" ]; then + IMAGE_NAME=$1 + fi + echo "building a new image for the container [$IMAGE_NAME]" + docker build -t ${IMAGE_NAME} . + return $? +} + +function generate_docs { + if [ "$2" != "" ]; then + IMAGE_NAME=$2 + LOCAL_VOLUME=$1 + else + if [ "$1" != "" ]; then + LOCAL_VOLUME=$1 + fi + fi + echo "generating the documents using the docker image [$LOCAL_VOLUME] [$IMAGE_NAME]" + if [ ! -d ${LOCAL_VOLUME} ]; then + mkdir -p ${LOCAL_VOLUME} || { + echo failed to generate local volume dir at ${LOCAL_VOLUME} + return 1 + } + fi + docker run --rm -t -v ${LOCAL_VOLUME}:/tmp/docs/content/commands \ + ${IMAGE_NAME} ./generate_docs.sh -o /tmp/docs/content/commands + return $? +} + + +if [ $# -lt 1 ]; then + echo "usage: -b|--build [tagname] | -r|--run [image name]" + exit 1 +fi + +while [[ $# -gt 0 ]]; do + case $1 in + -b|--build) + build_image $2 + exit $? + ;; + -r|--run) + generate_docs $2 $3 + exit $? + ;; + *) + echo "usage: -b|--build [tagname] | -r|--run [image name]" + exit 1 + ;; + esac +done + diff --git a/scripts/generate_commands_docs.py b/scripts/generate_commands_docs.py new file mode 100755 index 00000000..493c1761 --- /dev/null +++ b/scripts/generate_commands_docs.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python3 + +# Copyright 2022, DragonflyDB authors. All rights reserved. +# See LICENSE for licensing terms. + + +import pathlib +import requests +import sys +import json +import argparse +import logging +from functools import partial +from io import StringIO, TextIOWrapper +from random import choice +import re +import os +from textwrap import fill +from typing import Optional, Sequence +from xmlrpc.client import Boolean +from enum import Enum +from railroad import * +from typing import List +import pytoml +import yaml + +USER = "dragonflydb" +REPO_NAME = "redis-doc" + +# Non-breaking space +NBSP = '\xa0' + +# HTML Word Break Opportunity +WBR = '' + + +class ArgumentType(Enum): + INTEGER = 'integer' + DOUBLE = 'double' + STRING = 'string' + UNIX_TIME = 'unix-time' + PATTERN = 'pattern' + KEY = 'key' + ONEOF = 'oneof' + BLOCK = 'block' + PURE_TOKEN = 'pure-token' + COMMAND = 'command' + + +FM_TYPES = { + '{\n': { + 'eof': '}\n', + 'ext': '.json' + }, + '---\n': { + 'eof': '---\n', + 'ext': '.yaml' + }, + '+++\n': { + 'eof': '+++\n', + 'ext': '.toml' + } +} + +PARSERS = { + '.json': { + 'dump': lambda x, y: json.dump(x, y, indent=4), + 'dumps': lambda x: json.dumps(x, indent=4), + 'load': lambda x: json.load(x), + 'loads': lambda x: json.loads(x), + }, + '.yaml': { + 'dump': lambda x, y: yaml.dump(x, y), + 'dumps': lambda x: yaml.dump(x), + 'load': lambda x: yaml.load(x, Loader=yaml.FullLoader), + 'loads': lambda x: yaml.load(io.StringIO(x), Loader=yaml.FullLoader), + }, + '.toml': { + 'dump': lambda x, y: pytoml.dump(x, y), + 'dumps': lambda x: pytoml.dumps(x), + 'load': lambda x: pytoml.load(x), + 'loads': lambda x: pytoml.loads(x), + }, +} + + +def do_dumps(payload: str, file_type: str): + return PARSERS[file_type]['dumps'](payload) + + +def do_dump_file(payload: str, file_type: str, file_handler: TextIOWrapper): + return PARSERS[file_type]['dump'](payload, file_handler) + + +def do_load(payload: str, file_type: str): + return PARSERS[file_type]["loads"](payload) + + +def do_load_file(payload: str, file_type: str, file_handler: TextIOWrapper): + PARSERS[file_type]["load"](payload, file_handler) + + +def command_filename(name: str) -> str: + return name.lower().replace(' ', '-') + + +def base_url(user, repo_name): + return f'https://raw.githubusercontent.com/{user}/{repo_name}/master/' + + +def read_from_github(path_to_file): + base_name = base_url(USER, REPO_NAME) + url = f'{base_name}/{path_to_file}' + req = requests.get(url) + if req.status_code == requests.codes.ok: + return req.text + else: + logging.debug('Content was not found.') + return None + + +def read_from_local(base_path, path_to_file): + full_path = os.path.join(base_path, path_to_file) + txt = pathlib.Path(full_path).read_text() + return txt + + +def read_md_file(command_name: str, read_f) -> str: + command_name = command_filename(command_name) + content = read_f(f"commands/{command_name}.md") + return content + + +def read_commands_json(read_f) -> dict: + content = read_f('commands.json') + if content is not None: + commands = json.loads(content) + return commands + else: + return None + + +def generate_payload_from_md(file_name: str, read_f): + payload = read_md_file(file_name, read_f) + i = 0 + while i < len(payload): + if payload[i].startswith('\ufeff'): # BOM workaround + payload[i] = payload[i][1:] + if payload[i].strip() == '': # Munch newlines and whitespaces + i += 1 + else: + fm_type = FM_TYPES.get(payload[i]) + break + + if not fm_type: + payload = ''.join(payload) + fm_type = FM_TYPES.get('---\n') + fm_ext = fm_type.get('ext') + return payload, fm_type, None + eof, fm_ext = fm_type.get('eof'), fm_type.get('ext') + + fm_data = {} + if fm_ext == '.json': + fm_data.update(do_load( + fm_ext, ''.join(payload[i:j+1]))) + payload = ''.join(payload[j+1:]) + else: + fm_data.update(do_load( + fm_ext, ''.join(payload[i+1:j]))) + payload = ''.join(payload[j+1:]) + logging.info( + f"we have payload of size {len(payload)} and fm type {fm_type}") + return payload, fm_type, fm_data + + +class RuntimeException(Exception): + def __init__(self, err: str): + self.error = err + + def __str__(self): + return self.error + + +# def syntax(name: str, **kwargs) -> str: +# opts = { +# 'width': kwargs.get('width', 68), +# 'subsequent_indent': ' ' * 2, +# 'break_long_words': False, +# 'break_on_hyphens': False +# } +# args = [name] + [arg.syntax() for arg in self._arguments] +# return fill(' '.join(args), **opts) + + +class Argument: + def __init__(self, data: dict = {}, level: int = 0) -> None: + self._level: int = level + self._name: str = data['name'] + self._type = ArgumentType(data['type']) + self._optional: bool = data.get('optional', False) + self._multiple: bool = data.get('multiple', False) + self._multiple_token: bool = data.get('multiple_token', False) + self._token: str | None = data.get('token') + self._display: str = data.get('display', self._name) + if self._token == '': + self._token = '""' + self._arguments: List[Argument] = [ + Argument(arg, self._level+1) for arg in data.get('arguments', [])] + + def syntax(self, **kwargs) -> str: + show_types = kwargs.get('show_types') + args = '' + if self._type == ArgumentType.BLOCK: + args += ' '.join([arg.syntax() for arg in self._arguments]) + elif self._type == ArgumentType.ONEOF: + args += f' | '.join([arg.syntax() for arg in self._arguments]) + elif self._type != ArgumentType.PURE_TOKEN: + args += self._display + if show_types: + args += f':{self._type.value}' + + syntax = '' + if self._optional: + syntax += '[' + + if self._token: + syntax += f'{self._token}' + if self._type != ArgumentType.PURE_TOKEN: + syntax += NBSP + + if self._type == ArgumentType.ONEOF and (not self._optional or self._token): + syntax += '<' + + if self._multiple: + if self._multiple_token: + syntax += f'{args} [{self._token} {args} ...]' + else: + syntax += f'{args} [{args} ...]' + else: + syntax += args + + if self._type == ArgumentType.ONEOF and (not self._optional or self._token): + syntax = f'{syntax.rstrip()}>' + if self._optional: + syntax = f'{syntax.rstrip()}]' + + return f'{syntax}' + + +def make_args_list(name: str, args: dict) -> Argument: + carg = { + 'name': name, + 'type': ArgumentType.COMMAND.value, + 'arguments': args.get('arguments', []) + } + arguments = Argument(carg, 0) + return arguments + + +def args_syntax(name: str, args: dict) -> str: + arguments = make_args_list(name, args) + s = ' '.join([arg.syntax() for arg in arguments._arguments[1:]]) + return s + + +def help_command(name: str) -> bool: + return name.endswith(" HELP") + + +def container_cmd(cmd_info: dict) -> bool: + return cmd_info.get('arguments') is None and cmd_info.get('arity', 0) == -2 and len(cmd_info.split(' ')) == 1 + + +def command_args(name: str, args: dict, **kwargs) -> str: + opts = { + 'width': kwargs.get('width', 68), + 'subsequent_indent': ' ' * 2, + 'break_long_words': False, + 'break_on_hyphens': False + } + arguments = make_args_list(name, args) + args = [name] + [arg.syntax() for arg in arguments._arguments] + return fill(' '.join(args), **opts) + + +def get_command_tokens(arguments: dict) -> set: + rep = set() + if type(arguments) is list: + for arg in arguments: + rep = rep.union(get_command_tokens(arg)) + else: + if 'token' in arguments: + rep.add(arguments['token']) + if 'arguments' in arguments: + for arg in arguments['arguments']: + rep = rep.union(get_command_tokens(arg)) + return rep + + +def generate_commands_links(name: str, commands: dict, payload: str) -> str: + def fix_links(commands: dict, name: str): + if name: + exclude = set([name]) + tokens = get_command_tokens(commands.get(name)) + exclude.union(tokens) + else: + exclude = set() + + def generate_md_links(m): + command = m.group(1) + if command in commands and command not in exclude: + return f'[`{command}`](/commands/{command_filename(command)})' + else: + return m.group(0) + return generate_md_links + links = fix_links(commands, name) + rep = re.sub(r'`([A-Z][A-Z-_ \.]*)`', links, payload) + rep = re.sub(r'`!([A-Z][A-Z-_ \.]*)`', lambda x: f'`{x[1]}`', rep) + return rep + + +def add_command_frontmatter(name: str, commands: dict, fm_data: dict): + """ Sets a JSON FrontMatter payload for a command page """ + data = commands.get(name) + data.update({ + 'title': name, + 'linkTitle': name, + 'description': data.get('summary'), + 'syntax_str': args_syntax(name, data), + 'syntax_fmt': command_args(name, data), + 'hidden': container_cmd(data) or help_command(name) + }) + if 'replaced_by' in data: + data['replaced_by'] = generate_commands_links( + name, commands, data.get('replaced_by')) + fm_type = FM_TYPES.get('---\n') + fm_ext = fm_type.get('ext') + fm_data.update(data) + + +def persist(payload: str, data: dict, is_json: Boolean, filepath: str, file_type: dict) -> int: + ''' + Save to persist storage + ''' + fm = do_dumps(data, file_type['ext']) + logging.info( + f"saving payload of len {len(payload)} and header of {len(data)} to file {filepath}") + if not is_json: + fm = f'{file_type.get("eof")}{fm}{file_type.get("eof")}' + else: + fm += '\n' + + payload = fm + payload + dir_p = os.path.dirname(filepath) + logging.info( + f"saving payload of len {len(payload)} to file {filepath}, creating at {dir_p}") + pathlib.Path(dir_p).mkdir(parents=True, exist_ok=True) + with open(filepath, 'w') as f: + return f.write(payload) > 0 + + +def convert_command_sections(payload: str): + """ Converts redis-doc section headers to MD """ + rep = re.sub(r'@examples\n', + '## Examples\n', payload) + rep = re.sub(r'@return\n', + '## Return\n', rep) + return rep + + +def convert_reply_shortcuts(payload: str) -> str: + """ Convert RESP2 reply type shortcuts to links """ + def reply(x): + resp2 = { + 'nil': ('resp-bulk-strings', 'Null reply'), + 'simple-string': ('resp-simple-strings', 'Simple string reply'), + 'integer': ('resp-integers', 'Integer reply'), + 'bulk-string': ('resp-bulk-strings', 'Bulk string reply'), + 'array': ('resp-arrays', 'Array reply'), + 'error': ('resp-errors', 'Error reply'), + + } + rep = resp2.get(x.group(1), None) + if rep: + return f'[{rep[1]}](/docs/reference/protocol-spec#{rep[0]})' + return f'[]' + + rep = re.sub(r'@([a-z\-]+)-reply', reply, payload) + return rep + + +def process_command(name: str, commands: dict, payload: str) -> str: + """ New command processing logic """ + payload = generate_commands_links( + name, commands, payload) + payload = convert_command_sections(payload) + payload = convert_reply_shortcuts(payload) + # payload = convert_cli_snippets(payload) + return payload + + +def persist_command(name: str, commands: dict, payload: str, filepath: str, fm_data: dict, file_type: dict) -> int: + """ + name: the name of the command: for example "ACL" + commands: data taken from data/commands.json file and convert to python dict + filepath: path to the index.md file: content/en/commands/ + fm_data: dict with {'github_branch': , 'github_path': ', github_repo': } + is_json: str: + def command_filename(name: str) -> str: + return name.lower().replace(' ', '-') + """ + get the path to the md file from the commands list based on the command name + """ + file_name = command_filename(cmd_name) + return os.path.join(base_name, file_name) + + +def driver(commands: dict, out_dir: str, read_f, support_cmd: List) -> int: + # try: + commands = read_commands_json(read_f) + max = len(support_cmd) + count = 0 + for cmd_name, cmd_info in commands.items(): + if cmd_name in support_cmd: + if count == max: + logging.info(f"finish processing after {count}") + return 0 + else: + count += 1 + if read_md_file(command_filename(cmd_name), read_function): + logging.debug(f"successfully read data for {cmd_name}") + else: + logging.error(f"failed to read the md file for {cmd_name}") + return False + filepath = os.path.join(filepath_from_cmd( + cmd_name, out_dir), "index.md") + default_repos = {'github_branch': "main", + "github_path": filepath, "github_repo": f"https://{USER}/{REPO_NAME}"} + payload, f_type, m_data = generate_payload_from_md( + cmd_name, read_f) + fm_data = m_data if m_data else default_repos + fm_type = f_type if f_type is not None else FM_TYPES['---\n'] + if persist_command(cmd_name, commands, payload, filepath, fm_data, fm_type) == 0: + raise RuntimeException( + f"failed to process command {cmd_name} for file {filepath}") + return 0 + # except Exception as e: + # print( + # f"failed to process commands into documentation: {e}") + # return -1 + + +def process_docs(read_f, out_dir) -> bool: + commands = read_commands_json(read_function) + if commands is None: + logging.error("failed to read commands list") + return False + else: + logging.debug( + f"we have {len(commands)} commands from commands.json file") + cmd = ["HSET", "LLEN", "LMPOP", "LMOVE", + "SET", "GET", "SUBSCRIBE", "PUBLISH"] + if driver(commands, out_dir, read_f, cmd) < 0: + return False + return True + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.DEBUG, format=f'{sys.argv[0]}: %(levelname)s %(asctime)s %(message)s', filename="/tmp/build_commands_documents_messages.log") + parser = argparse.ArgumentParser( + description='converting markdown files to website commands documentation.') + parser.add_argument( + "-l", "--local", help="Running with local repository ", type=str) + # + parser.add_argument( + "-o", "--output_dir", help="Output directory to which result will be writing ", type=str, required=True) + args = parser.parse_args() + if args.local: + if os.path.exists(args.local) and pathlib.Path(args.local).is_dir(): + read_f = partial(read_from_local, args.local) + logging.debug(f"running from files at {args.local}") + read_function = read_f + else: + print( + f"not such directory {args.local} - you must use a valid path to the location of the commands.json and commands directory") + sys.exit(1) + else: + logging.debug(f"running with remote host {base_url(USER, REPO_NAME)}") + read_function = read_from_github + # + p = pathlib.Path(args.output_dir) + p.mkdir(parents=True, exist_ok=True) + if not process_docs(read_function, args.output_dir): + logging.error("failed to process input data to commands documents") + sys.exit(1) + sys.exit(0) diff --git a/scripts/generate_docs.sh b/scripts/generate_docs.sh new file mode 100755 index 00000000..d7497e09 --- /dev/null +++ b/scripts/generate_docs.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash + +# Copyright 2022, DragonflyDB authors. All rights reserved. +# See LICENSE for licensing terms. + +## This would generate files that would be used for the web site as documentation for the supported commands +## The build process is based on 2 components: +## 1. The input files taken from repo https://github.com/dragonflydb/redis-doc +## 2. The python script generate_commands_docs.py found under this directory +## Command line options: +## --clone This would clone the files from https://github.com/dragonflydb/redis-doc.git into local directory and run on it (the default) +## --remote This would use the above URL to fatch files withtout copying them locally +## -- output This would place the generated files at the default to /tmp//docs/content +## --local Run this using a local files at location +## --clean This would remove the build and log files +## Please note that the log from the python script is at /tmp/build_commands_documents_messages + +function generate_docs { + # create python virtual env for this + cd ${ABS_SCRIPT_PATH} + virtualenv docenv || { + echo "failed to create virtual env to run python script" + return 1 + } + ./docenv/bin/pip3 install -r requirements.txt || { + echo "failed to install dependencies for the python script" + rm -rf docenv + return 1 + } + mkdir -p ${OUTPUT_DIR} || { + rm -rf docenv + echo "failed to generate output directory at ${OUTPUT_DIR}" + return 1 + } + if [ "${RUN_LOCAL}" = "no" ]; then + ./docenv/bin/python3 generate_commands_docs.py --output_dir ${OUTPUT_DIR} + else + ./docenv/bin/python3 generate_commands_docs.py --output_dir ${OUTPUT_DIR} --local ${INPUT_DIR} + fi + result=$? + rm -rf ./docenv + return ${result} + +} + +function do_cleanup { + echo "cleaninng up" + rm -rf ${GIT_CLONED_PATH} /tmp/build_commands_documents_messages.log ${OUTPUT_DIR} + echo "finish cleaning ${GIT_CLONED_PATH} tmp/build_commands_documents_messages.log and ${OUTPUT_DIR}" + return 0 +} +function do_git_clone { + current_dir=${PWD} + echo "cloning into ${GIT_CLONED_PATH}" + if [ ! -d ${GIT_CLONED_PATH}/redis-doc ]; then + mkdir -p ${GIT_CLONED_PATH} || { + echo "failed to create ${GIT_CLONED_PATH} to clone files into" + return 1 + } + cd ${GIT_CLONED_PATH} && git clone https://github.com/dragonflydb/redis-doc.git || { + echo "failed to clone from redis docs" + return 1 + } + else + cd ${GIT_CLONED_PATH}/redis-doc && git pull || { + echo "failed to update docs repo" + return 1 + } + fi + cd ${current_dir} + +} + +################# MAIN ######################################################## + +CURRENT_RELATIVE=`dirname "$0"` +ABS_SCRIPT_PATH=`( cd "$CURRENT_RELATIVE" && pwd )` + +# Default values +CURRENT_DATE=$(date +%s) +OUTPUT_DIR=/tmp/${CURRENT_DATE}/docs/content +GIT_CLONED_PATH=/tmp/build_commands_docs/docs_repo +RUN_LOCAL="yes" +DO_CLONE="yes" +INPUT_DIR=${GIT_CLONED_PATH}/redis-doc + +## Process command line args +while [[ $# -gt 0 ]]; do + case $1 in + -c|--clone) + DO_CLONE="yes" + RUN_LOCAL="yes" + INPUT_DIR=${GIT_CLONED_PATH}/redis-doc + shift + ;; + -r|--remote) + RUN_LOCAL="no" + DO_CLONE="no" + shift + ;; + -o|--output) + OUTPUT_DIR=$2 + shift + shift + ;; + -l|--local) + RUN_LOCAL="yes" + DO_CLONE="no" + INPUT_DIR=$2 + shift + shift + ;; + -c|--clean) + DO_DELETE="yes" + shift + ;; + *) + echo "usage: [-l|--local ] [-c|--clone (default option)] [-r|--remote] [-o|--output ] [-d|--delete]" + exit 1 + ;; + esac +done + +echo "Running cleanup ${DO_DELETE}, cloning ${DO_CLONE}, runing on local files ${RUN_LOCAL}, output will be writing to ${OUTPUT_DIR} taking files from ${INPUT_DIR}" + +## Run +if [ "${DO_DELETE}" = "yes" ]; then + do_cleanup + exit 0 +fi + + +if [ "${DO_CLONE}" = "yes" ]; then + do_git_clone + if [ $? -ne 0 ]; then + echo "failed to clone doc repo locally" + exit 1 + else + echo "successfully cloned doc repo to ${INPUT_DIR}" + fi + +fi + +generate_docs + +if [ $? -ne 0 ]; then + echo "failed to generate docs" + exit 1 +else + echo "successfully generated docs to ${OUTPUT_DIR}" + exit 0 +fi + diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 00000000..66306d27 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,5 @@ +pytoml==0.1.21 +PyYAML==6.0 +railroad-diagrams==1.1.1 +requests==2.27.1 +semver==2.13.0