From b24449a9070a7ee749dd795ea74ea2d8f6186f71 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 29 Oct 2014 15:36:13 -0700 Subject: [PATCH 001/161] simplify initial prettify phase in EMCC_DEBUG=2 --- emcc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/emcc b/emcc index 41f91840f73c7..b28b7634ee2f7 100755 --- a/emcc +++ b/emcc @@ -1394,7 +1394,8 @@ try: else: for i in range(len(js_optimizer_queue)): name = js_optimizer_queue[i] - passes = add_opt_args([name]) + passes = [name] if name is not 'pretty' else [] + passes = add_opt_args(passes) if shared.Settings.ASM_JS: passes = ['asm'] + passes just_split = False @@ -1421,9 +1422,7 @@ try: if DEBUG == '2': # Clean up the syntax a bit - final = shared.Building.js_optimizer(final, [], jcache, debug_level >= 4) - js_transform_tempfiles.append(final) - if DEBUG: save_intermediate('pretty') + js_optimizer_queue += ['pretty'] def get_eliminate(): if shared.Settings.ALLOW_MEMORY_GROWTH: From 1381c289d56725aca566225d7f7761e17daa2e53 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 29 Oct 2014 17:16:02 -0700 Subject: [PATCH 002/161] REMOVE use JSON in EMCC_DEBUG=2, and time results --- emcc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/emcc b/emcc index b28b7634ee2f7..3f6642714ab7a 100755 --- a/emcc +++ b/emcc @@ -1402,15 +1402,17 @@ try: just_concat = False # XXX experimental code to emit JSON in intermediate phases - #if i > 0: - # passes = ['receiveJSON'] + passes - # just_split = True - #if i < len(js_optimizer_queue)-1: - # passes += ['emitJSON'] - # just_concat = True + if i > 0: + passes = ['receiveJSON'] + passes + just_split = True + if i < len(js_optimizer_queue)-1: + passes += ['emitJSON'] + just_concat = True logging.debug('applying js optimization pass: %s', passes) + t = time.time() final = shared.Building.js_optimizer(final, passes, jcache, debug_level >= 4, js_optimizer_extra_info, just_split=just_split, just_concat=just_concat) + print >> sys.stderr, 'took', time.time()-t js_transform_tempfiles.append(final) save_intermediate(name) js_optimizer_queue_history += js_optimizer_queue From 64db81a58f78b58ba8ec6e872289005d9967d9e2 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 29 Oct 2014 17:16:31 -0700 Subject: [PATCH 003/161] add rapidjson --- tools/optimizer/rapidjson/allocators.h | 246 +++ tools/optimizer/rapidjson/document.h | 1908 +++++++++++++++++ tools/optimizer/rapidjson/encodedstream.h | 290 +++ tools/optimizer/rapidjson/encodings.h | 630 ++++++ tools/optimizer/rapidjson/error/en.h | 71 + tools/optimizer/rapidjson/error/error.h | 150 ++ tools/optimizer/rapidjson/filereadstream.h | 94 + tools/optimizer/rapidjson/filestream.h | 73 + tools/optimizer/rapidjson/filewritestream.h | 97 + tools/optimizer/rapidjson/internal/dtoa.h | 418 ++++ tools/optimizer/rapidjson/internal/itoa.h | 306 +++ tools/optimizer/rapidjson/internal/meta.h | 189 ++ tools/optimizer/rapidjson/internal/pow10.h | 59 + tools/optimizer/rapidjson/internal/stack.h | 183 ++ tools/optimizer/rapidjson/internal/strfunc.h | 43 + tools/optimizer/rapidjson/memorybuffer.h | 76 + tools/optimizer/rapidjson/memorystream.h | 67 + .../optimizer/rapidjson/msinttypes/inttypes.h | 306 +++ tools/optimizer/rapidjson/msinttypes/stdint.h | 296 +++ tools/optimizer/rapidjson/prettywriter.h | 205 ++ tools/optimizer/rapidjson/rapidjson.h | 567 +++++ tools/optimizer/rapidjson/reader.h | 1369 ++++++++++++ tools/optimizer/rapidjson/stringbuffer.h | 79 + tools/optimizer/rapidjson/writer.h | 389 ++++ 24 files changed, 8111 insertions(+) create mode 100644 tools/optimizer/rapidjson/allocators.h create mode 100644 tools/optimizer/rapidjson/document.h create mode 100644 tools/optimizer/rapidjson/encodedstream.h create mode 100644 tools/optimizer/rapidjson/encodings.h create mode 100644 tools/optimizer/rapidjson/error/en.h create mode 100644 tools/optimizer/rapidjson/error/error.h create mode 100644 tools/optimizer/rapidjson/filereadstream.h create mode 100644 tools/optimizer/rapidjson/filestream.h create mode 100644 tools/optimizer/rapidjson/filewritestream.h create mode 100644 tools/optimizer/rapidjson/internal/dtoa.h create mode 100644 tools/optimizer/rapidjson/internal/itoa.h create mode 100644 tools/optimizer/rapidjson/internal/meta.h create mode 100644 tools/optimizer/rapidjson/internal/pow10.h create mode 100644 tools/optimizer/rapidjson/internal/stack.h create mode 100644 tools/optimizer/rapidjson/internal/strfunc.h create mode 100644 tools/optimizer/rapidjson/memorybuffer.h create mode 100644 tools/optimizer/rapidjson/memorystream.h create mode 100644 tools/optimizer/rapidjson/msinttypes/inttypes.h create mode 100644 tools/optimizer/rapidjson/msinttypes/stdint.h create mode 100644 tools/optimizer/rapidjson/prettywriter.h create mode 100644 tools/optimizer/rapidjson/rapidjson.h create mode 100644 tools/optimizer/rapidjson/reader.h create mode 100644 tools/optimizer/rapidjson/stringbuffer.h create mode 100644 tools/optimizer/rapidjson/writer.h diff --git a/tools/optimizer/rapidjson/allocators.h b/tools/optimizer/rapidjson/allocators.h new file mode 100644 index 0000000000000..c99485e5080f4 --- /dev/null +++ b/tools/optimizer/rapidjson/allocators.h @@ -0,0 +1,246 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_ALLOCATORS_H_ +#define RAPIDJSON_ALLOCATORS_H_ + +#include "rapidjson.h" + +namespace rapidjson { + +/////////////////////////////////////////////////////////////////////////////// +// Allocator + +/*! \class rapidjson::Allocator + \brief Concept for allocating, resizing and freeing memory block. + + Note that Malloc() and Realloc() are non-static but Free() is static. + + So if an allocator need to support Free(), it needs to put its pointer in + the header of memory block. + +\code +concept Allocator { + static const bool kNeedFree; //!< Whether this allocator needs to call Free(). + + // Allocate a memory block. + // \param size of the memory block in bytes. + // \returns pointer to the memory block. + void* Malloc(size_t size); + + // Resize a memory block. + // \param originalPtr The pointer to current memory block. Null pointer is permitted. + // \param originalSize The current size in bytes. (Design issue: since some allocator may not book-keep this, explicitly pass to it can save memory.) + // \param newSize the new size in bytes. + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize); + + // Free a memory block. + // \param pointer to the memory block. Null pointer is permitted. + static void Free(void *ptr); +}; +\endcode +*/ + +/////////////////////////////////////////////////////////////////////////////// +// CrtAllocator + +//! C-runtime library allocator. +/*! This class is just wrapper for standard C library memory routines. + \note implements Allocator concept +*/ +class CrtAllocator { +public: + static const bool kNeedFree = true; + void* Malloc(size_t size) { return std::malloc(size); } + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { (void)originalSize; return std::realloc(originalPtr, newSize); } + static void Free(void *ptr) { std::free(ptr); } +}; + +/////////////////////////////////////////////////////////////////////////////// +// MemoryPoolAllocator + +//! Default memory allocator used by the parser and DOM. +/*! This allocator allocate memory blocks from pre-allocated memory chunks. + + It does not free memory blocks. And Realloc() only allocate new memory. + + The memory chunks are allocated by BaseAllocator, which is CrtAllocator by default. + + User may also supply a buffer as the first chunk. + + If the user-buffer is full then additional chunks are allocated by BaseAllocator. + + The user-buffer is not deallocated by this allocator. + + \tparam BaseAllocator the allocator type for allocating memory chunks. Default is CrtAllocator. + \note implements Allocator concept +*/ +template +class MemoryPoolAllocator { +public: + static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) + + //! Constructor with chunkSize. + /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. + \param baseAllocator The allocator for allocating memory chunks. + */ + MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : + chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0) + { + if (!baseAllocator_) + ownBaseAllocator_ = baseAllocator_ = new BaseAllocator(); + AddChunk(chunk_capacity_); + } + + //! Constructor with user-supplied buffer. + /*! The user buffer will be used firstly. When it is full, memory pool allocates new chunk with chunk size. + + The user buffer will not be deallocated when this allocator is destructed. + + \param buffer User supplied buffer. + \param size Size of the buffer in bytes. It must at least larger than sizeof(ChunkHeader). + \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. + \param baseAllocator The allocator for allocating memory chunks. + */ + MemoryPoolAllocator(void *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : + chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0) + { + RAPIDJSON_ASSERT(buffer != 0); + RAPIDJSON_ASSERT(size > sizeof(ChunkHeader)); + chunkHead_ = reinterpret_cast(buffer); + chunkHead_->capacity = size - sizeof(ChunkHeader); + chunkHead_->size = 0; + chunkHead_->next = 0; + } + + //! Destructor. + /*! This deallocates all memory chunks, excluding the user-supplied buffer. + */ + ~MemoryPoolAllocator() { + Clear(); + delete ownBaseAllocator_; + } + + //! Deallocates all memory chunks, excluding the user-supplied buffer. + void Clear() { + while(chunkHead_ != 0 && chunkHead_ != userBuffer_) { + ChunkHeader* next = chunkHead_->next; + baseAllocator_->Free(chunkHead_); + chunkHead_ = next; + } + } + + //! Computes the total capacity of allocated memory chunks. + /*! \return total capacity in bytes. + */ + size_t Capacity() const { + size_t capacity = 0; + for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) + capacity += c->capacity; + return capacity; + } + + //! Computes the memory blocks allocated. + /*! \return total used bytes. + */ + size_t Size() const { + size_t size = 0; + for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) + size += c->size; + return size; + } + + //! Allocates a memory block. (concept Allocator) + void* Malloc(size_t size) { + size = RAPIDJSON_ALIGN(size); + if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) + AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size); + + void *buffer = reinterpret_cast(chunkHead_ + 1) + chunkHead_->size; + chunkHead_->size += size; + return buffer; + } + + //! Resizes a memory block (concept Allocator) + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { + if (originalPtr == 0) + return Malloc(newSize); + + // Do not shrink if new size is smaller than original + if (originalSize >= newSize) + return originalPtr; + + // Simply expand it if it is the last allocation and there is sufficient space + if (originalPtr == (char *)(chunkHead_ + 1) + chunkHead_->size - originalSize) { + size_t increment = static_cast(newSize - originalSize); + increment = RAPIDJSON_ALIGN(increment); + if (chunkHead_->size + increment <= chunkHead_->capacity) { + chunkHead_->size += increment; + return originalPtr; + } + } + + // Realloc process: allocate and copy memory, do not free original buffer. + void* newBuffer = Malloc(newSize); + RAPIDJSON_ASSERT(newBuffer != 0); // Do not handle out-of-memory explicitly. + return std::memcpy(newBuffer, originalPtr, originalSize); + } + + //! Frees a memory block (concept Allocator) + static void Free(void *ptr) { (void)ptr; } // Do nothing + +private: + //! Copy constructor is not permitted. + MemoryPoolAllocator(const MemoryPoolAllocator& rhs) /* = delete */; + //! Copy assignment operator is not permitted. + MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) /* = delete */; + + //! Creates a new chunk. + /*! \param capacity Capacity of the chunk in bytes. + */ + void AddChunk(size_t capacity) { + ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(sizeof(ChunkHeader) + capacity)); + chunk->capacity = capacity; + chunk->size = 0; + chunk->next = chunkHead_; + chunkHead_ = chunk; + } + + static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity. + + //! Chunk header for perpending to each chunk. + /*! Chunks are stored as a singly linked list. + */ + struct ChunkHeader { + size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). + size_t size; //!< Current size of allocated memory in bytes. + ChunkHeader *next; //!< Next chunk in the linked list. + }; + + ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation. + size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. + void *userBuffer_; //!< User supplied buffer. + BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. + BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object. +}; + +} // namespace rapidjson + +#endif // RAPIDJSON_ENCODINGS_H_ diff --git a/tools/optimizer/rapidjson/document.h b/tools/optimizer/rapidjson/document.h new file mode 100644 index 0000000000000..0f7974738f6b6 --- /dev/null +++ b/tools/optimizer/rapidjson/document.h @@ -0,0 +1,1908 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_DOCUMENT_H_ +#define RAPIDJSON_DOCUMENT_H_ + +/*! \file document.h */ + +#include "reader.h" +#include "internal/meta.h" +#include "internal/strfunc.h" +#include // placement new + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +#elif defined(__GNUC__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_HAS_STDSTRING + +#ifndef RAPIDJSON_HAS_STDSTRING +#ifdef RAPIDJSON_DOXYGEN_RUNNING +#define RAPIDJSON_HAS_STDSTRING 1 // force generation of documentation +#else +#define RAPIDJSON_HAS_STDSTRING 0 // no std::string support by default +#endif +/*! \def RAPIDJSON_HAS_STDSTRING + \ingroup RAPIDJSON_CONFIG + \brief Enable RapidJSON support for \c std::string + + By defining this preprocessor symbol to \c 1, several convenience functions for using + \ref rapidjson::GenericValue with \c std::string are enabled, especially + for construction and comparison. + + \hideinitializer +*/ +#include +#endif // RAPIDJSON_HAS_STDSTRING + +#ifndef RAPIDJSON_NOMEMBERITERATORCLASS +#include // std::iterator, std::random_access_iterator_tag +#endif + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS +#include // std::move +#endif + +namespace rapidjson { + +// Forward declaration. +template +class GenericValue; + +//! Name-value pair in a JSON object value. +/*! + This class was internal to GenericValue. It used to be a inner struct. + But a compiler (IBM XL C/C++ for AIX) have reported to have problem with that so it moved as a namespace scope struct. + https://code.google.com/p/rapidjson/issues/detail?id=64 +*/ +template +struct GenericMember { + GenericValue name; //!< name of member (must be a string) + GenericValue value; //!< value of member. +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericMemberIterator + +#ifndef RAPIDJSON_NOMEMBERITERATORCLASS + +//! (Constant) member iterator for a JSON object value +/*! + \tparam Const Is this a constant iterator? + \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) + \tparam Allocator Allocator type for allocating memory of object, array and string. + + This class implements a Random Access Iterator for GenericMember elements + of a GenericValue, see ISO/IEC 14882:2003(E) C++ standard, 24.1 [lib.iterator.requirements]. + + \note This iterator implementation is mainly intended to avoid implicit + conversions from iterator values to \c NULL, + e.g. from GenericValue::FindMember. + + \note Define \c RAPIDJSON_NOMEMBERITERATORCLASS to fall back to a + pointer-based implementation, if your platform doesn't provide + the C++ header. + + \see GenericMember, GenericValue::MemberIterator, GenericValue::ConstMemberIterator + */ +template +class GenericMemberIterator + : public std::iterator >::Type> { + + friend class GenericValue; + template friend class GenericMemberIterator; + + typedef GenericMember PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef std::iterator BaseType; + +public: + //! Iterator type itself + typedef GenericMemberIterator Iterator; + //! Constant iterator type + typedef GenericMemberIterator ConstIterator; + //! Non-constant iterator type + typedef GenericMemberIterator NonConstIterator; + + //! Pointer to (const) GenericMember + typedef typename BaseType::pointer Pointer; + //! Reference to (const) GenericMember + typedef typename BaseType::reference Reference; + //! Signed integer type (e.g. \c ptrdiff_t) + typedef typename BaseType::difference_type DifferenceType; + + //! Default constructor (singular value) + /*! Creates an iterator pointing to no element. + \note All operations, except for comparisons, are undefined on such values. + */ + GenericMemberIterator() : ptr_() {} + + //! Iterator conversions to more const + /*! + \param it (Non-const) iterator to copy from + + Allows the creation of an iterator from another GenericMemberIterator + that is "less const". Especially, creating a non-constant iterator + from a constant iterator are disabled: + \li const -> non-const (not ok) + \li const -> const (ok) + \li non-const -> const (ok) + \li non-const -> non-const (ok) + + \note If the \c Const template parameter is already \c false, this + constructor effectively defines a regular copy-constructor. + Otherwise, the copy constructor is implicitly defined. + */ + GenericMemberIterator(const NonConstIterator & it) : ptr_(it.ptr_) {} + + //! @name stepping + //@{ + Iterator& operator++(){ ++ptr_; return *this; } + Iterator& operator--(){ --ptr_; return *this; } + Iterator operator++(int){ Iterator old(*this); ++ptr_; return old; } + Iterator operator--(int){ Iterator old(*this); --ptr_; return old; } + //@} + + //! @name increment/decrement + //@{ + Iterator operator+(DifferenceType n) const { return Iterator(ptr_+n); } + Iterator operator-(DifferenceType n) const { return Iterator(ptr_-n); } + + Iterator& operator+=(DifferenceType n) { ptr_+=n; return *this; } + Iterator& operator-=(DifferenceType n) { ptr_-=n; return *this; } + //@} + + //! @name relations + //@{ + bool operator==(ConstIterator that) const { return ptr_ == that.ptr_; } + bool operator!=(ConstIterator that) const { return ptr_ != that.ptr_; } + bool operator<=(ConstIterator that) const { return ptr_ <= that.ptr_; } + bool operator>=(ConstIterator that) const { return ptr_ >= that.ptr_; } + bool operator< (ConstIterator that) const { return ptr_ < that.ptr_; } + bool operator> (ConstIterator that) const { return ptr_ > that.ptr_; } + //@} + + //! @name dereference + //@{ + Reference operator*() const { return *ptr_; } + Pointer operator->() const { return ptr_; } + Reference operator[](DifferenceType n) const { return ptr_[n]; } + //@} + + //! Distance + DifferenceType operator-(ConstIterator that) const { return ptr_-that.ptr_; } + +private: + //! Internal constructor from plain pointer + explicit GenericMemberIterator(Pointer p) : ptr_(p) {} + + Pointer ptr_; //!< raw pointer +}; + +#else // RAPIDJSON_NOMEMBERITERATORCLASS + +// class-based member iterator implementation disabled, use plain pointers + +template +struct GenericMemberIterator; + +//! non-const GenericMemberIterator +template +struct GenericMemberIterator { + //! use plain pointer as iterator type + typedef GenericMember* Iterator; +}; +//! const GenericMemberIterator +template +struct GenericMemberIterator { + //! use plain const pointer as iterator type + typedef const GenericMember* Iterator; +}; + +#endif // RAPIDJSON_NOMEMBERITERATORCLASS + +/////////////////////////////////////////////////////////////////////////////// +// GenericStringRef + +//! Reference to a constant string (not taking a copy) +/*! + \tparam CharType character type of the string + + This helper class is used to automatically infer constant string + references for string literals, especially from \c const \b (!) + character arrays. + + The main use is for creating JSON string values without copying the + source string via an \ref Allocator. This requires that the referenced + string pointers have a sufficient lifetime, which exceeds the lifetime + of the associated GenericValue. + + \b Example + \code + Value v("foo"); // ok, no need to copy & calculate length + const char foo[] = "foo"; + v.SetString(foo); // ok + + const char* bar = foo; + // Value x(bar); // not ok, can't rely on bar's lifetime + Value x(StringRef(bar)); // lifetime explicitly guaranteed by user + Value y(StringRef(bar, 3)); // ok, explicitly pass length + \endcode + + \see StringRef, GenericValue::SetString +*/ +template +struct GenericStringRef { + typedef CharType Ch; //!< character type of the string + + //! Create string reference from \c const character array + /*! + This constructor implicitly creates a constant string reference from + a \c const character array. It has better performance than + \ref StringRef(const CharType*) by inferring the string \ref length + from the array length, and also supports strings containing null + characters. + + \tparam N length of the string, automatically inferred + + \param str Constant character array, lifetime assumed to be longer + than the use of the string in e.g. a GenericValue + + \post \ref s == str + + \note Constant complexity. + \note There is a hidden, private overload to disallow references to + non-const character arrays to be created via this constructor. + By this, e.g. function-scope arrays used to be filled via + \c snprintf are excluded from consideration. + In such cases, the referenced string should be \b copied to the + GenericValue instead. + */ + template + GenericStringRef(const CharType (&str)[N]) RAPIDJSON_NOEXCEPT + : s(str), length(N-1) {} + + //! Explicitly create string reference from \c const character pointer + /*! + This constructor can be used to \b explicitly create a reference to + a constant string pointer. + + \see StringRef(const CharType*) + + \param str Constant character pointer, lifetime assumed to be longer + than the use of the string in e.g. a GenericValue + + \post \ref s == str + + \note There is a hidden, private overload to disallow references to + non-const character arrays to be created via this constructor. + By this, e.g. function-scope arrays used to be filled via + \c snprintf are excluded from consideration. + In such cases, the referenced string should be \b copied to the + GenericValue instead. + */ + explicit GenericStringRef(const CharType* str) + : s(str), length(internal::StrLen(str)){ RAPIDJSON_ASSERT(s != NULL); } + + //! Create constant string reference from pointer and length + /*! \param str constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \param len length of the string, excluding the trailing NULL terminator + + \post \ref s == str && \ref length == len + \note Constant complexity. + */ + GenericStringRef(const CharType* str, SizeType len) + : s(str), length(len) { RAPIDJSON_ASSERT(s != NULL); } + + //! implicit conversion to plain CharType pointer + operator const Ch *() const { return s; } + + const Ch* const s; //!< plain CharType pointer + const SizeType length; //!< length of the string (excluding the trailing NULL terminator) + +private: + //! Disallow copy-assignment + GenericStringRef operator=(const GenericStringRef&); + //! Disallow construction from non-const array + template + GenericStringRef(CharType (&str)[N]) /* = delete */; +}; + +//! Mark a character pointer as constant string +/*! Mark a plain character pointer as a "string literal". This function + can be used to avoid copying a character string to be referenced as a + value in a JSON GenericValue object, if the string's lifetime is known + to be valid long enough. + \tparam CharType Character type of the string + \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \return GenericStringRef string reference object + \relatesalso GenericStringRef + + \see GenericValue::GenericValue(StringRefType), GenericValue::operator=(StringRefType), GenericValue::SetString(StringRefType), GenericValue::PushBack(StringRefType, Allocator&), GenericValue::AddMember +*/ +template +inline GenericStringRef StringRef(const CharType* str) { + return GenericStringRef(str, internal::StrLen(str)); +} + +//! Mark a character pointer as constant string +/*! Mark a plain character pointer as a "string literal". This function + can be used to avoid copying a character string to be referenced as a + value in a JSON GenericValue object, if the string's lifetime is known + to be valid long enough. + + This version has better performance with supplied length, and also + supports string containing null characters. + + \tparam CharType character type of the string + \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \param length The length of source string. + \return GenericStringRef string reference object + \relatesalso GenericStringRef +*/ +template +inline GenericStringRef StringRef(const CharType* str, size_t length) { + return GenericStringRef(str, SizeType(length)); +} + +#if RAPIDJSON_HAS_STDSTRING +//! Mark a string object as constant string +/*! Mark a string object (e.g. \c std::string) as a "string literal". + This function can be used to avoid copying a string to be referenced as a + value in a JSON GenericValue object, if the string's lifetime is known + to be valid long enough. + + \tparam CharType character type of the string + \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \return GenericStringRef string reference object + \relatesalso GenericStringRef + \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. +*/ +template +inline GenericStringRef StringRef(const std::basic_string& str) { + return GenericStringRef(str.data(), SizeType(str.size())); +} +#endif + +/////////////////////////////////////////////////////////////////////////////// +// GenericValue type traits +namespace internal { + +template +struct IsGenericValueImpl : FalseType {}; + +// select candidates according to nested encoding and allocator types +template struct IsGenericValueImpl::Type, typename Void::Type> + : IsBaseOf, T>::Type {}; + +// helper to match arbitrary GenericValue instantiations, including derived classes +template struct IsGenericValue : IsGenericValueImpl::Type {}; + +} // namespace internal + +/////////////////////////////////////////////////////////////////////////////// +// GenericValue + +//! Represents a JSON value. Use Value for UTF8 encoding and default allocator. +/*! + A JSON value can be one of 7 types. This class is a variant type supporting + these types. + + Use the Value if UTF8 and default allocator + + \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) + \tparam Allocator Allocator type for allocating memory of object, array and string. +*/ +template > +class GenericValue { +public: + //! Name-value pair in an object. + typedef GenericMember Member; + typedef Encoding EncodingType; //!< Encoding type from template parameter. + typedef Allocator AllocatorType; //!< Allocator type from template parameter. + typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. + typedef GenericStringRef StringRefType; //!< Reference to a constant string + typedef typename GenericMemberIterator::Iterator MemberIterator; //!< Member iterator for iterating in object. + typedef typename GenericMemberIterator::Iterator ConstMemberIterator; //!< Constant member iterator for iterating in object. + typedef GenericValue* ValueIterator; //!< Value iterator for iterating in array. + typedef const GenericValue* ConstValueIterator; //!< Constant value iterator for iterating in array. + + //!@name Constructors and destructor. + //@{ + + //! Default constructor creates a null value. + GenericValue() RAPIDJSON_NOEXCEPT : data_(), flags_(kNullFlag) {} + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_), flags_(rhs.flags_) { + rhs.flags_ = kNullFlag; // give up contents + } +#endif + +private: + //! Copy constructor is not permitted. + GenericValue(const GenericValue& rhs); + +public: + + //! Constructor with JSON value type. + /*! This creates a Value of specified type with default content. + \param type Type of the value. + \note Default content for number is zero. + */ + explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_(), flags_() { + static const unsigned defaultFlags[7] = { + kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kConstStringFlag, + kNumberAnyFlag + }; + RAPIDJSON_ASSERT(type <= kNumberType); + flags_ = defaultFlags[type]; + } + + //! Explicit copy constructor (with allocator) + /*! Creates a copy of a Value by using the given Allocator + \tparam SourceAllocator allocator of \c rhs + \param rhs Value to copy from (read-only) + \param allocator Allocator for allocating copied elements and buffers. Commonly use GenericDocument::GetAllocator(). + \see CopyFrom() + */ + template< typename SourceAllocator > + GenericValue(const GenericValue& rhs, Allocator & allocator); + + //! Constructor for boolean value. + /*! \param b Boolean value + \note This constructor is limited to \em real boolean values and rejects + implicitly converted types like arbitrary pointers. Use an explicit cast + to \c bool, if you want to construct a boolean JSON value in such cases. + */ +#ifndef RAPIDJSON_DOXYGEN_RUNNING // hide SFINAE from Doxygen + template + explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame))) RAPIDJSON_NOEXCEPT +#else + explicit GenericValue(bool b) RAPIDJSON_NOEXCEPT +#endif + : data_(), flags_(b ? kTrueFlag : kFalseFlag) { + // safe-guard against failing SFINAE + RAPIDJSON_STATIC_ASSERT((internal::IsSame::Value)); + } + + //! Constructor for int value. + explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberIntFlag) { + data_.n.i64 = i; + if (i >= 0) + flags_ |= kUintFlag | kUint64Flag; + } + + //! Constructor for unsigned value. + explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberUintFlag) { + data_.n.u64 = u; + if (!(u & 0x80000000)) + flags_ |= kIntFlag | kInt64Flag; + } + + //! Constructor for int64_t value. + explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberInt64Flag) { + data_.n.i64 = i64; + if (i64 >= 0) { + flags_ |= kNumberUint64Flag; + if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) + flags_ |= kUintFlag; + if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + flags_ |= kIntFlag; + } + else if (i64 >= static_cast(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + flags_ |= kIntFlag; + } + + //! Constructor for uint64_t value. + explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberUint64Flag) { + data_.n.u64 = u64; + if (!(u64 & RAPIDJSON_UINT64_C2(0x80000000, 0x00000000))) + flags_ |= kInt64Flag; + if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) + flags_ |= kUintFlag; + if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + flags_ |= kIntFlag; + } + + //! Constructor for double value. + explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberDoubleFlag) { data_.n.d = d; } + + //! Constructor for constant string (i.e. do not make a copy of string) + GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_(), flags_() { SetStringRaw(StringRef(s, length)); } + + //! Constructor for constant string (i.e. do not make a copy of string) + explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_(), flags_() { SetStringRaw(s); } + + //! Constructor for copy-string (i.e. do make a copy of string) + GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s, length), allocator); } + + //! Constructor for copy-string (i.e. do make a copy of string) + GenericValue(const Ch*s, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s), allocator); } + +#if RAPIDJSON_HAS_STDSTRING + //! Constructor for copy-string from a string object (i.e. do make a copy of string) + /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + GenericValue(const std::basic_string& s, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s), allocator); } +#endif + + //! Destructor. + /*! Need to destruct elements of array, members of object, or copy-string. + */ + ~GenericValue() { + if (Allocator::kNeedFree) { // Shortcut by Allocator's trait + switch(flags_) { + case kArrayFlag: + for (GenericValue* v = data_.a.elements; v != data_.a.elements + data_.a.size; ++v) + v->~GenericValue(); + Allocator::Free(data_.a.elements); + break; + + case kObjectFlag: + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + Allocator::Free(data_.o.members); + break; + + case kCopyStringFlag: + Allocator::Free(const_cast(data_.s.str)); + break; + + default: + break; // Do nothing for other types. + } + } + } + + //@} + + //!@name Assignment operators + //@{ + + //! Assignment with move semantics. + /*! \param rhs Source of the assignment. It will become a null value after assignment. + */ + GenericValue& operator=(GenericValue& rhs) RAPIDJSON_NOEXCEPT { + RAPIDJSON_ASSERT(this != &rhs); + this->~GenericValue(); + RawAssign(rhs); + return *this; + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move assignment in C++11 + GenericValue& operator=(GenericValue&& rhs) RAPIDJSON_NOEXCEPT { + return *this = rhs.Move(); + } +#endif + + //! Assignment of constant string reference (no copy) + /*! \param str Constant string reference to be assigned + \note This overload is needed to avoid clashes with the generic primitive type assignment overload below. + \see GenericStringRef, operator=(T) + */ + GenericValue& operator=(StringRefType str) RAPIDJSON_NOEXCEPT { + GenericValue s(str); + return *this = s; + } + + //! Assignment with primitive types. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param value The value to be assigned. + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref SetString(const Ch*, Allocator&) (for copying) or + \ref StringRef() (to explicitly mark the pointer as constant) instead. + All other pointer types would implicitly convert to \c bool, + use \ref SetBool() instead. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::IsPointer), (GenericValue&)) + operator=(T value) { + GenericValue v(value); + return *this = v; + } + + //! Deep-copy assignment from Value + /*! Assigns a \b copy of the Value to the current Value object + \tparam SourceAllocator Allocator type of \c rhs + \param rhs Value to copy from (read-only) + \param allocator Allocator to use for copying + */ + template + GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator) { + RAPIDJSON_ASSERT((void*)this != (void const*)&rhs); + this->~GenericValue(); + new (this) GenericValue(rhs, allocator); + return *this; + } + + //! Exchange the contents of this value with those of other. + /*! + \param other Another value. + \note Constant complexity. + */ + GenericValue& Swap(GenericValue& other) RAPIDJSON_NOEXCEPT { + GenericValue temp; + temp.RawAssign(*this); + RawAssign(other); + other.RawAssign(temp); + return *this; + } + + //! Prepare Value for move semantics + /*! \return *this */ + GenericValue& Move() RAPIDJSON_NOEXCEPT { return *this; } + //@} + + //!@name Equal-to and not-equal-to operators + //@{ + //! Equal-to operator + /*! + \note If an object contains duplicated named member, comparing equality with any object is always \c false. + \note Linear time complexity (number of all values in the subtree and total lengths of all strings). + */ + template + bool operator==(const GenericValue& rhs) const { + typedef GenericValue RhsType; + if (GetType() != rhs.GetType()) + return false; + + switch (GetType()) { + case kObjectType: // Warning: O(n^2) inner-loop + if (data_.o.size != rhs.data_.o.size) + return false; + for (ConstMemberIterator lhsMemberItr = MemberBegin(); lhsMemberItr != MemberEnd(); ++lhsMemberItr) { + typename RhsType::ConstMemberIterator rhsMemberItr = rhs.FindMember(lhsMemberItr->name); + if (rhsMemberItr == rhs.MemberEnd() || lhsMemberItr->value != rhsMemberItr->value) + return false; + } + return true; + + case kArrayType: + if (data_.a.size != rhs.data_.a.size) + return false; + for (SizeType i = 0; i < data_.a.size; i++) + if ((*this)[i] != rhs[i]) + return false; + return true; + + case kStringType: + return StringEqual(rhs); + + case kNumberType: + if (IsDouble() || rhs.IsDouble()) + return GetDouble() == rhs.GetDouble(); // May convert one operand from integer to double. + else + return data_.n.u64 == rhs.data_.n.u64; + + default: // kTrueType, kFalseType, kNullType + return true; + } + } + + //! Equal-to operator with const C-string pointer + bool operator==(const Ch* rhs) const { return *this == GenericValue(StringRef(rhs)); } + +#if RAPIDJSON_HAS_STDSTRING + //! Equal-to operator with string object + /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + bool operator==(const std::basic_string& rhs) const { return *this == GenericValue(StringRef(rhs)); } +#endif + + //! Equal-to operator with primitive types + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c true, \c false + */ + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr,internal::IsGenericValue >), (bool)) operator==(const T& rhs) const { return *this == GenericValue(rhs); } + + //! Not-equal-to operator + /*! \return !(*this == rhs) + */ + template + bool operator!=(const GenericValue& rhs) const { return !(*this == rhs); } + + //! Not-equal-to operator with const C-string pointer + bool operator!=(const Ch* rhs) const { return !(*this == rhs); } + + //! Not-equal-to operator with arbitrary types + /*! \return !(*this == rhs) + */ + template RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& rhs) const { return !(*this == rhs); } + + //! Equal-to operator with arbitrary types (symmetric version) + /*! \return (rhs == lhs) + */ + template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator==(const T& lhs, const GenericValue& rhs) { return rhs == lhs; } + + //! Not-Equal-to operator with arbitrary types (symmetric version) + /*! \return !(rhs == lhs) + */ + template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& lhs, const GenericValue& rhs) { return !(rhs == lhs); } + //@} + + //!@name Type + //@{ + + Type GetType() const { return static_cast(flags_ & kTypeMask); } + bool IsNull() const { return flags_ == kNullFlag; } + bool IsFalse() const { return flags_ == kFalseFlag; } + bool IsTrue() const { return flags_ == kTrueFlag; } + bool IsBool() const { return (flags_ & kBoolFlag) != 0; } + bool IsObject() const { return flags_ == kObjectFlag; } + bool IsArray() const { return flags_ == kArrayFlag; } + bool IsNumber() const { return (flags_ & kNumberFlag) != 0; } + bool IsInt() const { return (flags_ & kIntFlag) != 0; } + bool IsUint() const { return (flags_ & kUintFlag) != 0; } + bool IsInt64() const { return (flags_ & kInt64Flag) != 0; } + bool IsUint64() const { return (flags_ & kUint64Flag) != 0; } + bool IsDouble() const { return (flags_ & kDoubleFlag) != 0; } + bool IsString() const { return (flags_ & kStringFlag) != 0; } + + //@} + + //!@name Null + //@{ + + GenericValue& SetNull() { this->~GenericValue(); new (this) GenericValue(); return *this; } + + //@} + + //!@name Bool + //@{ + + bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return flags_ == kTrueFlag; } + //!< Set boolean value + /*! \post IsBool() == true */ + GenericValue& SetBool(bool b) { this->~GenericValue(); new (this) GenericValue(b); return *this; } + + //@} + + //!@name Object + //@{ + + //! Set this value as an empty object. + /*! \post IsObject() == true */ + GenericValue& SetObject() { this->~GenericValue(); new (this) GenericValue(kObjectType); return *this; } + + //! Get the number of members in the object. + SizeType MemberCount() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size; } + + //! Check whether the object is empty. + bool ObjectEmpty() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size == 0; } + + //! Get a value from an object associated with the name. + /*! \pre IsObject() == true + \tparam T Either \c Ch or \c const \c Ch (template used for disambiguation with \ref operator[](SizeType)) + \note In version 0.1x, if the member is not found, this function returns a null value. This makes issue 7. + Since 0.2, if the name is not correct, it will assert. + If user is unsure whether a member exists, user should use HasMember() first. + A better approach is to use FindMember(). + \note Linear time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(GenericValue&)) operator[](T* name) { + GenericValue n(StringRef(name)); + return (*this)[n]; + } + template + RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(const GenericValue&)) operator[](T* name) const { return const_cast(*this)[name]; } + + //! Get a value from an object associated with the name. + /*! \pre IsObject() == true + \tparam SourceAllocator Allocator of the \c name value + + \note Compared to \ref operator[](T*), this version is faster because it does not need a StrLen(). + And it can also handle strings with embedded null characters. + + \note Linear time complexity. + */ + template + GenericValue& operator[](const GenericValue& name) { + MemberIterator member = FindMember(name); + if (member != MemberEnd()) + return member->value; + else { + RAPIDJSON_ASSERT(false); // see above note + static GenericValue NullValue; + return NullValue; + } + } + template + const GenericValue& operator[](const GenericValue& name) const { return const_cast(*this)[name]; } + + //! Const member iterator + /*! \pre IsObject() == true */ + ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(data_.o.members); } + //! Const \em past-the-end member iterator + /*! \pre IsObject() == true */ + ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(data_.o.members + data_.o.size); } + //! Member iterator + /*! \pre IsObject() == true */ + MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(data_.o.members); } + //! \em Past-the-end member iterator + /*! \pre IsObject() == true */ + MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(data_.o.members + data_.o.size); } + + //! Check whether a member exists in the object. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Whether a member with that name exists. + \note It is better to use FindMember() directly if you need the obtain the value as well. + \note Linear time complexity. + */ + bool HasMember(const Ch* name) const { return FindMember(name) != MemberEnd(); } + + //! Check whether a member exists in the object with GenericValue name. + /*! + This version is faster because it does not need a StrLen(). It can also handle string with null character. + \param name Member name to be searched. + \pre IsObject() == true + \return Whether a member with that name exists. + \note It is better to use FindMember() directly if you need the obtain the value as well. + \note Linear time complexity. + */ + template + bool HasMember(const GenericValue& name) const { return FindMember(name) != MemberEnd(); } + + //! Find member by name. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Iterator to member, if it exists. + Otherwise returns \ref MemberEnd(). + + \note Earlier versions of Rapidjson returned a \c NULL pointer, in case + the requested member doesn't exist. For consistency with e.g. + \c std::map, this has been changed to MemberEnd() now. + \note Linear time complexity. + */ + MemberIterator FindMember(const Ch* name) { + GenericValue n(StringRef(name)); + return FindMember(n); + } + + ConstMemberIterator FindMember(const Ch* name) const { return const_cast(*this).FindMember(name); } + + //! Find member by name. + /*! + This version is faster because it does not need a StrLen(). It can also handle string with null character. + \param name Member name to be searched. + \pre IsObject() == true + \return Iterator to member, if it exists. + Otherwise returns \ref MemberEnd(). + + \note Earlier versions of Rapidjson returned a \c NULL pointer, in case + the requested member doesn't exist. For consistency with e.g. + \c std::map, this has been changed to MemberEnd() now. + \note Linear time complexity. + */ + template + MemberIterator FindMember(const GenericValue& name) { + RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(name.IsString()); + MemberIterator member = MemberBegin(); + for ( ; member != MemberEnd(); ++member) + if (name.StringEqual(member->name)) + break; + return member; + } + template ConstMemberIterator FindMember(const GenericValue& name) const { return const_cast(*this).FindMember(name); } + + //! Add a member (name-value pair) to the object. + /*! \param name A string value as name of member. + \param value Value of any type. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note The ownership of \c name and \c value will be transferred to this object on success. + \pre IsObject() && name.IsString() + \post name.IsNull() && value.IsNull() + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { + RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(name.IsString()); + + Object& o = data_.o; + if (o.size >= o.capacity) { + if (o.capacity == 0) { + o.capacity = kDefaultObjectCapacity; + o.members = reinterpret_cast(allocator.Malloc(o.capacity * sizeof(Member))); + } + else { + SizeType oldCapacity = o.capacity; + o.capacity += (oldCapacity + 1) / 2; // grow by factor 1.5 + o.members = reinterpret_cast(allocator.Realloc(o.members, oldCapacity * sizeof(Member), o.capacity * sizeof(Member))); + } + } + o.members[o.size].name.RawAssign(name); + o.members[o.size].value.RawAssign(value); + o.size++; + return *this; + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericValue& AddMember(GenericValue&& name, GenericValue&& value, Allocator& allocator) { + return AddMember(name, value, allocator); + } + GenericValue& AddMember(GenericValue&& name, GenericValue& value, Allocator& allocator) { + return AddMember(name, value, allocator); + } + GenericValue& AddMember(GenericValue& name, GenericValue&& value, Allocator& allocator) { + return AddMember(name, value, allocator); + } + GenericValue& AddMember(StringRefType name, GenericValue&& value, Allocator& allocator) { + GenericValue n(name); + return AddMember(n, value, allocator); + } +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + + + //! Add a member (name-value pair) to the object. + /*! \param name A constant string reference as name of member. + \param value Value of any type. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note The ownership of \c value will be transferred to this object on success. + \pre IsObject() + \post value.IsNull() + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(StringRefType name, GenericValue& value, Allocator& allocator) { + GenericValue n(name); + return AddMember(n, value, allocator); + } + + //! Add a constant string value as member (name-value pair) to the object. + /*! \param name A constant string reference as name of member. + \param value constant string reference as value of member. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + \note This overload is needed to avoid clashes with the generic primitive type AddMember(StringRefType,T,Allocator&) overload below. + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(StringRefType name, StringRefType value, Allocator& allocator) { + GenericValue v(value); + return AddMember(name, v, allocator); + } + + //! Add any primitive value as member (name-value pair) to the object. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param name A constant string reference as name of member. + \param value Value of primitive type \c T as value of member + \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref + AddMember(StringRefType, StringRefType, Allocator&). + All other pointer types would implicitly convert to \c bool, + use an explicit cast instead, if needed. + \note Amortized Constant time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) + AddMember(StringRefType name, T value, Allocator& allocator) { + GenericValue n(name); + GenericValue v(value); + return AddMember(n, v, allocator); + } + + //! Remove all members in the object. + /*! This function do not deallocate memory in the object, i.e. the capacity is unchanged. + \note Linear time complexity. + */ + void RemoveAllMembers() { + RAPIDJSON_ASSERT(IsObject()); + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + data_.o.size = 0; + } + + //! Remove a member in object by its name. + /*! \param name Name of member to be removed. + \return Whether the member existed. + \note This function may reorder the object members. Use \ref + EraseMember(ConstMemberIterator) if you need to preserve the + relative order of the remaining members. + \note Linear time complexity. + */ + bool RemoveMember(const Ch* name) { + GenericValue n(StringRef(name)); + return RemoveMember(n); + } + + template + bool RemoveMember(const GenericValue& name) { + MemberIterator m = FindMember(name); + if (m != MemberEnd()) { + RemoveMember(m); + return true; + } + else + return false; + } + + //! Remove a member in object by iterator. + /*! \param m member iterator (obtained by FindMember() or MemberBegin()). + \return the new iterator after removal. + \note This function may reorder the object members. Use \ref + EraseMember(ConstMemberIterator) if you need to preserve the + relative order of the remaining members. + \note Constant time complexity. + */ + MemberIterator RemoveMember(MemberIterator m) { + RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(data_.o.size > 0); + RAPIDJSON_ASSERT(data_.o.members != 0); + RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); + + MemberIterator last(data_.o.members + (data_.o.size - 1)); + if (data_.o.size > 1 && m != last) { + // Move the last one to this place + *m = *last; + } + else { + // Only one left, just destroy + m->~Member(); + } + --data_.o.size; + return m; + } + + //! Remove a member from an object by iterator. + /*! \param pos iterator to the member to remove + \pre IsObject() == true && \ref MemberBegin() <= \c pos < \ref MemberEnd() + \return Iterator following the removed element. + If the iterator \c pos refers to the last element, the \ref MemberEnd() iterator is returned. + \note This function preserves the relative order of the remaining object + members. If you do not need this, use the more efficient \ref RemoveMember(MemberIterator). + \note Linear time complexity. + */ + MemberIterator EraseMember(ConstMemberIterator pos) { + return EraseMember(pos, pos +1); + } + + //! Remove members in the range [first, last) from an object. + /*! \param first iterator to the first member to remove + \param last iterator following the last member to remove + \pre IsObject() == true && \ref MemberBegin() <= \c first <= \c last <= \ref MemberEnd() + \return Iterator following the last removed element. + \note This function preserves the relative order of the remaining object + members. + \note Linear time complexity. + */ + MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) { + RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(data_.o.size > 0); + RAPIDJSON_ASSERT(data_.o.members != 0); + RAPIDJSON_ASSERT(first >= MemberBegin()); + RAPIDJSON_ASSERT(first <= last); + RAPIDJSON_ASSERT(last <= MemberEnd()); + + MemberIterator pos = MemberBegin() + (first - MemberBegin()); + for (MemberIterator itr = pos; itr != last; ++itr) + itr->~Member(); + std::memmove(&*pos, &*last, (MemberEnd() - last) * sizeof(Member)); + data_.o.size -= (last - first); + return pos; + } + + //@} + + //!@name Array + //@{ + + //! Set this value as an empty array. + /*! \post IsArray == true */ + GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } + + //! Get the number of elements in array. + SizeType Size() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size; } + + //! Get the capacity of array. + SizeType Capacity() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.capacity; } + + //! Check whether the array is empty. + bool Empty() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size == 0; } + + //! Remove all elements in the array. + /*! This function do not deallocate memory in the array, i.e. the capacity is unchanged. + \note Linear time complexity. + */ + void Clear() { + RAPIDJSON_ASSERT(IsArray()); + for (SizeType i = 0; i < data_.a.size; ++i) + data_.a.elements[i].~GenericValue(); + data_.a.size = 0; + } + + //! Get an element from array by index. + /*! \pre IsArray() == true + \param index Zero-based index of element. + \see operator[](T*) + */ + GenericValue& operator[](SizeType index) { + RAPIDJSON_ASSERT(IsArray()); + RAPIDJSON_ASSERT(index < data_.a.size); + return data_.a.elements[index]; + } + const GenericValue& operator[](SizeType index) const { return const_cast(*this)[index]; } + + //! Element iterator + /*! \pre IsArray() == true */ + ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return data_.a.elements; } + //! \em Past-the-end element iterator + /*! \pre IsArray() == true */ + ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return data_.a.elements + data_.a.size; } + //! Constant element iterator + /*! \pre IsArray() == true */ + ConstValueIterator Begin() const { return const_cast(*this).Begin(); } + //! Constant \em past-the-end element iterator + /*! \pre IsArray() == true */ + ConstValueIterator End() const { return const_cast(*this).End(); } + + //! Request the array to have enough capacity to store elements. + /*! \param newCapacity The capacity that the array at least need to have. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note Linear time complexity. + */ + GenericValue& Reserve(SizeType newCapacity, Allocator &allocator) { + RAPIDJSON_ASSERT(IsArray()); + if (newCapacity > data_.a.capacity) { + data_.a.elements = (GenericValue*)allocator.Realloc(data_.a.elements, data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)); + data_.a.capacity = newCapacity; + } + return *this; + } + + //! Append a GenericValue at the end of the array. + /*! \param value Value to be appended. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \post value.IsNull() == true + \return The value itself for fluent API. + \note The ownership of \c value will be transferred to this array on success. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + \note Amortized constant time complexity. + */ + GenericValue& PushBack(GenericValue& value, Allocator& allocator) { + RAPIDJSON_ASSERT(IsArray()); + if (data_.a.size >= data_.a.capacity) + Reserve(data_.a.capacity == 0 ? kDefaultArrayCapacity : (data_.a.capacity + (data_.a.capacity + 1) / 2), allocator); + data_.a.elements[data_.a.size++].RawAssign(value); + return *this; + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericValue& PushBack(GenericValue&& value, Allocator& allocator) { + return PushBack(value, allocator); + } +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + + //! Append a constant string reference at the end of the array. + /*! \param value Constant string reference to be appended. + \param allocator Allocator for reallocating memory. It must be the same one used previously. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \return The value itself for fluent API. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + \note Amortized constant time complexity. + \see GenericStringRef + */ + GenericValue& PushBack(StringRefType value, Allocator& allocator) { + return (*this).template PushBack(value, allocator); + } + + //! Append a primitive value at the end of the array. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param value Value of primitive type T to be appended. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \return The value itself for fluent API. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref PushBack(GenericValue&, Allocator&) or \ref + PushBack(StringRefType, Allocator&). + All other pointer types would implicitly convert to \c bool, + use an explicit cast instead, if needed. + \note Amortized constant time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) + PushBack(T value, Allocator& allocator) { + GenericValue v(value); + return PushBack(v, allocator); + } + + //! Remove the last element in the array. + /*! + \note Constant time complexity. + */ + GenericValue& PopBack() { + RAPIDJSON_ASSERT(IsArray()); + RAPIDJSON_ASSERT(!Empty()); + data_.a.elements[--data_.a.size].~GenericValue(); + return *this; + } + + //! Remove an element of array by iterator. + /*! + \param pos iterator to the element to remove + \pre IsArray() == true && \ref Begin() <= \c pos < \ref End() + \return Iterator following the removed element. If the iterator pos refers to the last element, the End() iterator is returned. + \note Linear time complexity. + */ + ValueIterator Erase(ConstValueIterator pos) { + return Erase(pos, pos + 1); + } + + //! Remove elements in the range [first, last) of the array. + /*! + \param first iterator to the first element to remove + \param last iterator following the last element to remove + \pre IsArray() == true && \ref Begin() <= \c first <= \c last <= \ref End() + \return Iterator following the last removed element. + \note Linear time complexity. + */ + ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) { + RAPIDJSON_ASSERT(IsArray()); + RAPIDJSON_ASSERT(data_.a.size > 0); + RAPIDJSON_ASSERT(data_.a.elements != 0); + RAPIDJSON_ASSERT(first >= Begin()); + RAPIDJSON_ASSERT(first <= last); + RAPIDJSON_ASSERT(last <= End()); + ValueIterator pos = Begin() + (first - Begin()); + for (ValueIterator itr = pos; itr != last; ++itr) + itr->~GenericValue(); + std::memmove(pos, last, (End() - last) * sizeof(GenericValue)); + data_.a.size -= (last - first); + return pos; + } + + //@} + + //!@name Number + //@{ + + int GetInt() const { RAPIDJSON_ASSERT(flags_ & kIntFlag); return data_.n.i.i; } + unsigned GetUint() const { RAPIDJSON_ASSERT(flags_ & kUintFlag); return data_.n.u.u; } + int64_t GetInt64() const { RAPIDJSON_ASSERT(flags_ & kInt64Flag); return data_.n.i64; } + uint64_t GetUint64() const { RAPIDJSON_ASSERT(flags_ & kUint64Flag); return data_.n.u64; } + + double GetDouble() const { + RAPIDJSON_ASSERT(IsNumber()); + if ((flags_ & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. + if ((flags_ & kIntFlag) != 0) return data_.n.i.i; // int -> double + if ((flags_ & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double + if ((flags_ & kInt64Flag) != 0) return (double)data_.n.i64; // int64_t -> double (may lose precision) + RAPIDJSON_ASSERT((flags_ & kUint64Flag) != 0); return (double)data_.n.u64; // uint64_t -> double (may lose precision) + } + + GenericValue& SetInt(int i) { this->~GenericValue(); new (this) GenericValue(i); return *this; } + GenericValue& SetUint(unsigned u) { this->~GenericValue(); new (this) GenericValue(u); return *this; } + GenericValue& SetInt64(int64_t i64) { this->~GenericValue(); new (this) GenericValue(i64); return *this; } + GenericValue& SetUint64(uint64_t u64) { this->~GenericValue(); new (this) GenericValue(u64); return *this; } + GenericValue& SetDouble(double d) { this->~GenericValue(); new (this) GenericValue(d); return *this; } + + //@} + + //!@name String + //@{ + + const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return ((flags_ & kInlineStrFlag) ? data_.ss.str : data_.s.str); } + + //! Get the length of string. + /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). + */ + SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((flags_ & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } + + //! Set this value as a string without copying source string. + /*! This version has better performance with supplied length, and also support string containing null character. + \param s source string pointer. + \param length The length of source string, excluding the trailing null terminator. + \return The value itself for fluent API. + \post IsString() == true && GetString() == s && GetStringLength() == length + \see SetString(StringRefType) + */ + GenericValue& SetString(const Ch* s, SizeType length) { return SetString(StringRef(s, length)); } + + //! Set this value as a string without copying source string. + /*! \param s source string reference + \return The value itself for fluent API. + \post IsString() == true && GetString() == s && GetStringLength() == s.length + */ + GenericValue& SetString(StringRefType s) { this->~GenericValue(); SetStringRaw(s); return *this; } + + //! Set this value as a string by copying from source string. + /*! This version has better performance with supplied length, and also support string containing null character. + \param s source string. + \param length The length of source string, excluding the trailing null terminator. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length + */ + GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { this->~GenericValue(); SetStringRaw(StringRef(s, length), allocator); return *this; } + + //! Set this value as a string by copying from source string. + /*! \param s source string. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length + */ + GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(s, internal::StrLen(s), allocator); } + +#if RAPIDJSON_HAS_STDSTRING + //! Set this value as a string by copying from source string. + /*! \param s source string. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s.data() && strcmp(GetString(),s.data() == 0 && GetStringLength() == s.size() + \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(s.data(), s.size(), allocator); } +#endif + + //@} + + //! Generate events of this value to a Handler. + /*! This function adopts the GoF visitor pattern. + Typical usage is to output this JSON value as JSON text via Writer, which is a Handler. + It can also be used to deep clone this value via GenericDocument, which is also a Handler. + \tparam Handler type of handler. + \param handler An object implementing concept Handler. + */ + template + bool Accept(Handler& handler) const { + switch(GetType()) { + case kNullType: return handler.Null(); + case kFalseType: return handler.Bool(false); + case kTrueType: return handler.Bool(true); + + case kObjectType: + if (!handler.StartObject()) + return false; + for (ConstMemberIterator m = MemberBegin(); m != MemberEnd(); ++m) { + if (!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.flags_ & kCopyFlag) != 0)) + return false; + if (!m->value.Accept(handler)) + return false; + } + return handler.EndObject(data_.o.size); + + case kArrayType: + if (!handler.StartArray()) + return false; + for (GenericValue* v = data_.a.elements; v != data_.a.elements + data_.a.size; ++v) + if (!v->Accept(handler)) + return false; + return handler.EndArray(data_.a.size); + + case kStringType: + return handler.String(GetString(), GetStringLength(), (flags_ & kCopyFlag) != 0); + + case kNumberType: + if (IsInt()) return handler.Int(data_.n.i.i); + else if (IsUint()) return handler.Uint(data_.n.u.u); + else if (IsInt64()) return handler.Int64(data_.n.i64); + else if (IsUint64()) return handler.Uint64(data_.n.u64); + else return handler.Double(data_.n.d); + + default: + RAPIDJSON_ASSERT(false); + } + return false; + } + +private: + template friend class GenericValue; + template friend class GenericDocument; + + enum { + kBoolFlag = 0x100, + kNumberFlag = 0x200, + kIntFlag = 0x400, + kUintFlag = 0x800, + kInt64Flag = 0x1000, + kUint64Flag = 0x2000, + kDoubleFlag = 0x4000, + kStringFlag = 0x100000, + kCopyFlag = 0x200000, + kInlineStrFlag = 0x400000, + + // Initial flags of different types. + kNullFlag = kNullType, + kTrueFlag = kTrueType | kBoolFlag, + kFalseFlag = kFalseType | kBoolFlag, + kNumberIntFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag, + kNumberUintFlag = kNumberType | kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag, + kNumberInt64Flag = kNumberType | kNumberFlag | kInt64Flag, + kNumberUint64Flag = kNumberType | kNumberFlag | kUint64Flag, + kNumberDoubleFlag = kNumberType | kNumberFlag | kDoubleFlag, + kNumberAnyFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag, + kConstStringFlag = kStringType | kStringFlag, + kCopyStringFlag = kStringType | kStringFlag | kCopyFlag, + kShortStringFlag = kStringType | kStringFlag | kCopyFlag | kInlineStrFlag, + kObjectFlag = kObjectType, + kArrayFlag = kArrayType, + + kTypeMask = 0xFF // bitwise-and with mask of 0xFF can be optimized by compiler + }; + + static const SizeType kDefaultArrayCapacity = 16; + static const SizeType kDefaultObjectCapacity = 16; + + struct String { + const Ch* str; + SizeType length; + unsigned hashcode; //!< reserved + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + // implementation detail: ShortString can represent zero-terminated strings up to MaxSize chars + // (excluding the terminating zero) and store a value to determine the length of the contained + // string in the last character str[LenPos] by storing "MaxSize - length" there. If the string + // to store has the maximal length of MaxSize then str[LenPos] will be 0 and therefore act as + // the string terminator as well. For getting the string length back from that value just use + // "MaxSize - str[LenPos]". + // This allows to store 11-chars strings in 32-bit mode and 15-chars strings in 64-bit mode + // inline (for `UTF8`-encoded strings). + struct ShortString { + enum { MaxChars = sizeof(String) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; + Ch str[MaxChars]; + + inline static bool Usable(SizeType len) { return (MaxSize >= len); } + inline void SetLength(SizeType len) { str[LenPos] = (Ch)(MaxSize - len); } + inline SizeType GetLength() const { return (SizeType)(MaxSize - str[LenPos]); } + }; // at most as many bytes as "String" above => 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + // By using proper binary layout, retrieval of different integer types do not need conversions. + union Number { +#if RAPIDJSON_ENDIAN == RAPIDJSON_LITTLEENDIAN + struct I { + int i; + char padding[4]; + }i; + struct U { + unsigned u; + char padding2[4]; + }u; +#else + struct I { + char padding[4]; + int i; + }i; + struct U { + char padding2[4]; + unsigned u; + }u; +#endif + int64_t i64; + uint64_t u64; + double d; + }; // 8 bytes + + struct Object { + Member* members; + SizeType size; + SizeType capacity; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + struct Array { + GenericValue* elements; + SizeType size; + SizeType capacity; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + union Data { + String s; + ShortString ss; + Number n; + Object o; + Array a; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + // Initialize this value as array with initial data, without calling destructor. + void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) { + flags_ = kArrayFlag; + data_.a.elements = (GenericValue*)allocator.Malloc(count * sizeof(GenericValue)); + std::memcpy(data_.a.elements, values, count * sizeof(GenericValue)); + data_.a.size = data_.a.capacity = count; + } + + //! Initialize this value as object with initial data, without calling destructor. + void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) { + flags_ = kObjectFlag; + data_.o.members = (Member*)allocator.Malloc(count * sizeof(Member)); + std::memcpy(data_.o.members, members, count * sizeof(Member)); + data_.o.size = data_.o.capacity = count; + } + + //! Initialize this value as constant string, without calling destructor. + void SetStringRaw(StringRefType s) RAPIDJSON_NOEXCEPT { + flags_ = kConstStringFlag; + data_.s.str = s; + data_.s.length = s.length; + } + + //! Initialize this value as copy string with initial data, without calling destructor. + void SetStringRaw(StringRefType s, Allocator& allocator) { + Ch* str = NULL; + if(ShortString::Usable(s.length)) { + flags_ = kShortStringFlag; + data_.ss.SetLength(s.length); + str = data_.ss.str; + } else { + flags_ = kCopyStringFlag; + data_.s.length = s.length; + str = (Ch *)allocator.Malloc((s.length + 1) * sizeof(Ch)); + data_.s.str = str; + } + std::memcpy(str, s, s.length * sizeof(Ch)); + str[s.length] = '\0'; + } + + //! Assignment without calling destructor + void RawAssign(GenericValue& rhs) RAPIDJSON_NOEXCEPT { + data_ = rhs.data_; + flags_ = rhs.flags_; + rhs.flags_ = kNullFlag; + } + + template + bool StringEqual(const GenericValue& rhs) const { + RAPIDJSON_ASSERT(IsString()); + RAPIDJSON_ASSERT(rhs.IsString()); + + const SizeType len1 = GetStringLength(); + const SizeType len2 = rhs.GetStringLength(); + if(len1 != len2) { return false; } + + const Ch* const str1 = GetString(); + const Ch* const str2 = rhs.GetString(); + if(str1 == str2) { return true; } // fast path for constant string + + return (std::memcmp(str1, str2, sizeof(Ch) * len1) == 0); + } + + Data data_; + unsigned flags_; +}; + +//! GenericValue with UTF8 encoding +typedef GenericValue > Value; + +/////////////////////////////////////////////////////////////////////////////// +// GenericDocument + +//! A document for parsing JSON text as DOM. +/*! + \note implements Handler concept + \tparam Encoding Encoding for both parsing and string storage. + \tparam Allocator Allocator for allocating memory for the DOM + \tparam StackAllocator Allocator for allocating memory for stack during parsing. + \warning Although GenericDocument inherits from GenericValue, the API does \b not provide any virtual functions, especially no virtual destructor. To avoid memory leaks, do not \c delete a GenericDocument object via a pointer to a GenericValue. +*/ +template , typename StackAllocator = CrtAllocator> +class GenericDocument : public GenericValue { +public: + typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. + typedef GenericValue ValueType; //!< Value type of the document. + typedef Allocator AllocatorType; //!< Allocator type from template parameter. + + //! Constructor + /*! \param allocator Optional allocator for allocating memory. + \param stackCapacity Optional initial capacity of stack in bytes. + \param stackAllocator Optional allocator for allocating memory for stack. + */ + GenericDocument(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : + allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() + { + if (!allocator_) + ownAllocator_ = allocator_ = new Allocator(); + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericDocument(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT + : ValueType(std::move(rhs)), + allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + stack_(std::move(rhs.stack_)), + parseResult_(rhs.parseResult_) + { + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.parseResult_ = ParseResult(); + } +#endif + + ~GenericDocument() { + Destroy(); + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move assignment in C++11 + GenericDocument& operator=(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT + { + // The cast to ValueType is necessary here, because otherwise it would + // attempt to call GenericValue's templated assignment operator. + ValueType::operator=(std::forward(rhs)); + + // Calling the destructor here would prematurely call stack_'s destructor + Destroy(); + + allocator_ = rhs.allocator_; + ownAllocator_ = rhs.ownAllocator_; + stack_ = std::move(rhs.stack_); + parseResult_ = rhs.parseResult_; + + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.parseResult_ = ParseResult(); + + return *this; + } +#endif + + //!@name Parse from stream + //!@{ + + //! Parse JSON text from an input stream (with Encoding conversion) + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam SourceEncoding Encoding of input stream + \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) { + ValueType::SetNull(); // Remove existing root if exist + GenericReader reader(&GetAllocator()); + ClearStackOnExit scope(*this); + parseResult_ = reader.template Parse(is, *this); + if (parseResult_) { + RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object + this->RawAssign(*stack_.template Pop(1)); // Add this-> to prevent issue 13. + } + return *this; + } + + //! Parse JSON text from an input stream + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) { + return ParseStream(is); + } + + //! Parse JSON text from an input stream (with \ref kParseDefaultFlags) + /*! \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) { + return ParseStream(is); + } + //!@} + + //!@name Parse in-place from mutable string + //!@{ + + //! Parse JSON text from a mutable string (with Encoding conversion) + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam SourceEncoding Transcoding from input Encoding + \param str Mutable zero-terminated string to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseInsitu(Ch* str) { + GenericInsituStringStream s(str); + return ParseStream(s); + } + + //! Parse JSON text from a mutable string + /*! \tparam parseFlags Combination of \ref ParseFlag. + \param str Mutable zero-terminated string to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseInsitu(Ch* str) { + return ParseInsitu(str); + } + + //! Parse JSON text from a mutable string (with \ref kParseDefaultFlags) + /*! \param str Mutable zero-terminated string to be parsed. + \return The document itself for fluent API. + */ + GenericDocument& ParseInsitu(Ch* str) { + return ParseInsitu(str); + } + //!@} + + //!@name Parse from read-only string + //!@{ + + //! Parse JSON text from a read-only string (with Encoding conversion) + /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). + \tparam SourceEncoding Transcoding from input Encoding + \param str Read-only zero-terminated string to be parsed. + */ + template + GenericDocument& Parse(const Ch* str) { + RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); + GenericStringStream s(str); + return ParseStream(s); + } + + //! Parse JSON text from a read-only string + /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). + \param str Read-only zero-terminated string to be parsed. + */ + template + GenericDocument& Parse(const Ch* str) { + return Parse(str); + } + + //! Parse JSON text from a read-only string (with \ref kParseDefaultFlags) + /*! \param str Read-only zero-terminated string to be parsed. + */ + GenericDocument& Parse(const Ch* str) { + return Parse(str); + } + //!@} + + //!@name Handling parse errors + //!@{ + + //! Whether a parse error has occured in the last parsing. + bool HasParseError() const { return parseResult_.IsError(); } + + //! Get the \ref ParseErrorCode of last parsing. + ParseErrorCode GetParseError() const { return parseResult_.Code(); } + + //! Get the position of last parsing error in input, 0 otherwise. + size_t GetErrorOffset() const { return parseResult_.Offset(); } + + //!@} + + //! Get the allocator of this document. + Allocator& GetAllocator() { return *allocator_; } + + //! Get the capacity of stack in bytes. + size_t GetStackCapacity() const { return stack_.GetCapacity(); } + +private: + // clear stack on any exit from ParseStream, e.g. due to exception + struct ClearStackOnExit { + explicit ClearStackOnExit(GenericDocument& d) : d_(d) {} + ~ClearStackOnExit() { d_.ClearStack(); } + private: + ClearStackOnExit(const ClearStackOnExit&); + ClearStackOnExit& operator=(const ClearStackOnExit&); + GenericDocument& d_; + }; + + // callers of the following private Handler functions + template friend class GenericReader; // for parsing + template friend class GenericValue; // for deep copying + + // Implementation of Handler + bool Null() { new (stack_.template Push()) ValueType(); return true; } + bool Bool(bool b) { new (stack_.template Push()) ValueType(b); return true; } + bool Int(int i) { new (stack_.template Push()) ValueType(i); return true; } + bool Uint(unsigned i) { new (stack_.template Push()) ValueType(i); return true; } + bool Int64(int64_t i) { new (stack_.template Push()) ValueType(i); return true; } + bool Uint64(uint64_t i) { new (stack_.template Push()) ValueType(i); return true; } + bool Double(double d) { new (stack_.template Push()) ValueType(d); return true; } + + bool String(const Ch* str, SizeType length, bool copy) { + if (copy) + new (stack_.template Push()) ValueType(str, length, GetAllocator()); + else + new (stack_.template Push()) ValueType(str, length); + return true; + } + + bool StartObject() { new (stack_.template Push()) ValueType(kObjectType); return true; } + + bool Key(const Ch* str, SizeType length, bool copy) { return String(str, length, copy); } + + bool EndObject(SizeType memberCount) { + typename ValueType::Member* members = stack_.template Pop(memberCount); + stack_.template Top()->SetObjectRaw(members, (SizeType)memberCount, GetAllocator()); + return true; + } + + bool StartArray() { new (stack_.template Push()) ValueType(kArrayType); return true; } + + bool EndArray(SizeType elementCount) { + ValueType* elements = stack_.template Pop(elementCount); + stack_.template Top()->SetArrayRaw(elements, elementCount, GetAllocator()); + return true; + } + +private: + //! Prohibit assignment + GenericDocument& operator=(const GenericDocument&); + + void ClearStack() { + if (Allocator::kNeedFree) + while (stack_.GetSize() > 0) // Here assumes all elements in stack array are GenericValue (Member is actually 2 GenericValue objects) + (stack_.template Pop(1))->~ValueType(); + else + stack_.Clear(); + stack_.ShrinkToFit(); + } + + void Destroy() { + delete ownAllocator_; + } + + static const size_t kDefaultStackCapacity = 1024; + Allocator* allocator_; + Allocator* ownAllocator_; + internal::Stack stack_; + ParseResult parseResult_; +}; + +//! GenericDocument with UTF8 encoding +typedef GenericDocument > Document; + +// defined here due to the dependency on GenericDocument +template +template +inline +GenericValue::GenericValue(const GenericValue& rhs, Allocator& allocator) +{ + GenericDocument d(&allocator); + rhs.Accept(d); + RawAssign(*d.stack_.template Pop(1)); +} + +} // namespace rapidjson + +#if defined(_MSC_VER) || defined(__GNUC__) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_DOCUMENT_H_ diff --git a/tools/optimizer/rapidjson/encodedstream.h b/tools/optimizer/rapidjson/encodedstream.h new file mode 100644 index 0000000000000..9dc00c7afc32c --- /dev/null +++ b/tools/optimizer/rapidjson/encodedstream.h @@ -0,0 +1,290 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_ENCODEDSTREAM_H_ +#define RAPIDJSON_ENCODEDSTREAM_H_ + +#include "rapidjson.h" + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +namespace rapidjson { + +//! Input byte stream wrapper with a statically bound encoding. +/*! + \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. + \tparam InputByteStream Type of input byte stream. For example, FileReadStream. +*/ +template +class EncodedInputStream { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); +public: + typedef typename Encoding::Ch Ch; + + EncodedInputStream(InputByteStream& is) : is_(is) { + current_ = Encoding::TakeBOM(is_); + } + + Ch Peek() const { return current_; } + Ch Take() { Ch c = current_; current_ = Encoding::Take(is_); return c; } + size_t Tell() const { return is_.Tell(); } + + // Not implemented + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + EncodedInputStream(const EncodedInputStream&); + EncodedInputStream& operator=(const EncodedInputStream&); + + InputByteStream& is_; + Ch current_; +}; + +//! Output byte stream wrapper with statically bound encoding. +/*! + \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. + \tparam InputByteStream Type of input byte stream. For example, FileWriteStream. +*/ +template +class EncodedOutputStream { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); +public: + typedef typename Encoding::Ch Ch; + + EncodedOutputStream(OutputByteStream& os, bool putBOM = true) : os_(os) { + if (putBOM) + Encoding::PutBOM(os_); + } + + void Put(Ch c) { Encoding::Put(os_, c); } + void Flush() { os_.Flush(); } + + // Not implemented + Ch Peek() const { RAPIDJSON_ASSERT(false); } + Ch Take() { RAPIDJSON_ASSERT(false); } + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + EncodedOutputStream(const EncodedOutputStream&); + EncodedOutputStream& operator=(const EncodedOutputStream&); + + OutputByteStream& os_; +}; + +#define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x + +//! Input stream wrapper with dynamically bound encoding and automatic encoding detection. +/*! + \tparam CharType Type of character for reading. + \tparam InputByteStream type of input byte stream to be wrapped. +*/ +template +class AutoUTFInputStream { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); +public: + typedef CharType Ch; + + //! Constructor. + /*! + \param is input stream to be wrapped. + \param type UTF encoding type if it is not detected from the stream. + */ + AutoUTFInputStream(InputByteStream& is, UTFType type = kUTF8) : is_(&is), type_(type), hasBOM_(false) { + DetectType(); + static const TakeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Take) }; + takeFunc_ = f[type_]; + current_ = takeFunc_(*is_); + } + + UTFType GetType() const { return type_; } + bool HasBOM() const { return hasBOM_; } + + Ch Peek() const { return current_; } + Ch Take() { Ch c = current_; current_ = takeFunc_(*is_); return c; } + size_t Tell() const { return is_->Tell(); } + + // Not implemented + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + AutoUTFInputStream(const AutoUTFInputStream&); + AutoUTFInputStream& operator=(const AutoUTFInputStream&); + + // Detect encoding type with BOM or RFC 4627 + void DetectType() { + // BOM (Byte Order Mark): + // 00 00 FE FF UTF-32BE + // FF FE 00 00 UTF-32LE + // FE FF UTF-16BE + // FF FE UTF-16LE + // EF BB BF UTF-8 + + const unsigned char* c = (const unsigned char *)is_->Peek4(); + if (!c) + return; + + unsigned bom = c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24); + hasBOM_ = false; + if (bom == 0xFFFE0000) { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } + else if (bom == 0x0000FEFF) { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } + else if ((bom & 0xFFFF) == 0xFFFE) { type_ = kUTF16BE; hasBOM_ = true; is_->Take(); is_->Take(); } + else if ((bom & 0xFFFF) == 0xFEFF) { type_ = kUTF16LE; hasBOM_ = true; is_->Take(); is_->Take(); } + else if ((bom & 0xFFFFFF) == 0xBFBBEF) { type_ = kUTF8; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); } + + // RFC 4627: Section 3 + // "Since the first two characters of a JSON text will always be ASCII + // characters [RFC0020], it is possible to determine whether an octet + // stream is UTF-8, UTF-16 (BE or LE), or UTF-32 (BE or LE) by looking + // at the pattern of nulls in the first four octets." + // 00 00 00 xx UTF-32BE + // 00 xx 00 xx UTF-16BE + // xx 00 00 00 UTF-32LE + // xx 00 xx 00 UTF-16LE + // xx xx xx xx UTF-8 + + if (!hasBOM_) { + unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); + switch (pattern) { + case 0x08: type_ = kUTF32BE; break; + case 0x0A: type_ = kUTF16BE; break; + case 0x01: type_ = kUTF32LE; break; + case 0x05: type_ = kUTF16LE; break; + case 0x0F: type_ = kUTF8; break; + default: break; // Use type defined by user. + } + } + + // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. + switch (type_) { + case kUTF8: + // Do nothing + break; + case kUTF16LE: + case kUTF16BE: + RAPIDJSON_ASSERT(sizeof(Ch) >= 2); + break; + case kUTF32LE: + case kUTF32BE: + RAPIDJSON_ASSERT(sizeof(Ch) >= 4); + break; + default: + RAPIDJSON_ASSERT(false); // Invalid type + } + } + + typedef Ch (*TakeFunc)(InputByteStream& is); + InputByteStream* is_; + UTFType type_; + Ch current_; + TakeFunc takeFunc_; + bool hasBOM_; +}; + +//! Output stream wrapper with dynamically bound encoding and automatic encoding detection. +/*! + \tparam CharType Type of character for writing. + \tparam InputByteStream type of output byte stream to be wrapped. +*/ +template +class AutoUTFOutputStream { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); +public: + typedef CharType Ch; + + //! Constructor. + /*! + \param os output stream to be wrapped. + \param type UTF encoding type. + \param putBOM Whether to write BOM at the beginning of the stream. + */ + AutoUTFOutputStream(OutputByteStream& os, UTFType type, bool putBOM) : os_(&os), type_(type) { + // RUntime check whether the size of character type is sufficient. It only perform checks with assertion. + switch (type_) { + case kUTF16LE: + case kUTF16BE: + RAPIDJSON_ASSERT(sizeof(Ch) >= 2); + break; + case kUTF32LE: + case kUTF32BE: + RAPIDJSON_ASSERT(sizeof(Ch) >= 4); + break; + case kUTF8: + // Do nothing + break; + default: + RAPIDJSON_ASSERT(false); // Invalid UTFType + } + + static const PutFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Put) }; + putFunc_ = f[type_]; + + if (putBOM) + PutBOM(); + } + + UTFType GetType() const { return type_; } + + void Put(Ch c) { putFunc_(*os_, c); } + void Flush() { os_->Flush(); } + + // Not implemented + Ch Peek() const { RAPIDJSON_ASSERT(false); } + Ch Take() { RAPIDJSON_ASSERT(false); } + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + AutoUTFOutputStream(const AutoUTFOutputStream&); + AutoUTFOutputStream& operator=(const AutoUTFOutputStream&); + + void PutBOM() { + typedef void (*PutBOMFunc)(OutputByteStream&); + static const PutBOMFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(PutBOM) }; + f[type_](*os_); + } + + typedef void (*PutFunc)(OutputByteStream&, Ch); + + OutputByteStream* os_; + UTFType type_; + PutFunc putFunc_; +}; + +#undef RAPIDJSON_ENCODINGS_FUNC + +} // namespace rapidjson + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/tools/optimizer/rapidjson/encodings.h b/tools/optimizer/rapidjson/encodings.h new file mode 100644 index 0000000000000..5c7da3e048c69 --- /dev/null +++ b/tools/optimizer/rapidjson/encodings.h @@ -0,0 +1,630 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_ENCODINGS_H_ +#define RAPIDJSON_ENCODINGS_H_ + +#include "rapidjson.h" + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4244) // conversion from 'type1' to 'type2', possible loss of data +RAPIDJSON_DIAG_OFF(4702) // unreachable code +#elif defined(__GNUC__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +namespace rapidjson { + +/////////////////////////////////////////////////////////////////////////////// +// Encoding + +/*! \class rapidjson::Encoding + \brief Concept for encoding of Unicode characters. + +\code +concept Encoding { + typename Ch; //! Type of character. A "character" is actually a code unit in unicode's definition. + + enum { supportUnicode = 1 }; // or 0 if not supporting unicode + + //! \brief Encode a Unicode codepoint to an output stream. + //! \param os Output stream. + //! \param codepoint An unicode codepoint, ranging from 0x0 to 0x10FFFF inclusively. + template + static void Encode(OutputStream& os, unsigned codepoint); + + //! \brief Decode a Unicode codepoint from an input stream. + //! \param is Input stream. + //! \param codepoint Output of the unicode codepoint. + //! \return true if a valid codepoint can be decoded from the stream. + template + static bool Decode(InputStream& is, unsigned* codepoint); + + //! \brief Validate one Unicode codepoint from an encoded stream. + //! \param is Input stream to obtain codepoint. + //! \param os Output for copying one codepoint. + //! \return true if it is valid. + //! \note This function just validating and copying the codepoint without actually decode it. + template + static bool Validate(InputStream& is, OutputStream& os); + + // The following functions are deal with byte streams. + + //! Take a character from input byte stream, skip BOM if exist. + template + static CharType TakeBOM(InputByteStream& is); + + //! Take a character from input byte stream. + template + static Ch Take(InputByteStream& is); + + //! Put BOM to output byte stream. + template + static void PutBOM(OutputByteStream& os); + + //! Put a character to output byte stream. + template + static void Put(OutputByteStream& os, Ch c); +}; +\endcode +*/ + +/////////////////////////////////////////////////////////////////////////////// +// UTF8 + +//! UTF-8 encoding. +/*! http://en.wikipedia.org/wiki/UTF-8 + http://tools.ietf.org/html/rfc3629 + \tparam CharType Code unit for storing 8-bit UTF-8 data. Default is char. + \note implements Encoding concept +*/ +template +struct UTF8 { + typedef CharType Ch; + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + if (codepoint <= 0x7F) + os.Put(static_cast(codepoint & 0xFF)); + else if (codepoint <= 0x7FF) { + os.Put(static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint & 0x3F)))); + } + else if (codepoint <= 0xFFFF) { + os.Put(static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + os.Put(static_cast(0x80 | (codepoint & 0x3F))); + } + else { + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + os.Put(static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + os.Put(static_cast(0x80 | (codepoint & 0x3F))); + } + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { +#define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | ((unsigned char)c & 0x3Fu) +#define TRANS(mask) result &= ((GetRange((unsigned char)c) & mask) != 0) +#define TAIL() COPY(); TRANS(0x70) + Ch c = is.Take(); + if (!(c & 0x80)) { + *codepoint = (unsigned char)c; + return true; + } + + unsigned char type = GetRange((unsigned char)c); + *codepoint = (0xFF >> type) & (unsigned char)c; + bool result = true; + switch (type) { + case 2: TAIL(); return result; + case 3: TAIL(); TAIL(); return result; + case 4: COPY(); TRANS(0x50); TAIL(); return result; + case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; + case 6: TAIL(); TAIL(); TAIL(); return result; + case 10: COPY(); TRANS(0x20); TAIL(); return result; + case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; + default: return false; + } +#undef COPY +#undef TRANS +#undef TAIL + } + + template + static bool Validate(InputStream& is, OutputStream& os) { +#define COPY() os.Put(c = is.Take()) +#define TRANS(mask) result &= ((GetRange((unsigned char)c) & mask) != 0) +#define TAIL() COPY(); TRANS(0x70) + Ch c; + COPY(); + if (!(c & 0x80)) + return true; + + bool result = true; + switch (GetRange((unsigned char)c)) { + case 2: TAIL(); return result; + case 3: TAIL(); TAIL(); return result; + case 4: COPY(); TRANS(0x50); TAIL(); return result; + case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; + case 6: TAIL(); TAIL(); TAIL(); return result; + case 10: COPY(); TRANS(0x20); TAIL(); return result; + case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; + default: return false; + } +#undef COPY +#undef TRANS +#undef TAIL + } + + static unsigned char GetRange(unsigned char c) { + // Referring to DFA of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + // With new mapping 1 -> 0x10, 7 -> 0x20, 9 -> 0x40, such that AND operation can test multiple types. + static const unsigned char type[] = { + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, + 0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, + }; + return type[c]; + } + + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + Ch c = Take(is); + if ((unsigned char)c != 0xEFu) return c; + c = is.Take(); + if ((unsigned char)c != 0xBBu) return c; + c = is.Take(); + if ((unsigned char)c != 0xBFu) return c; + c = is.Take(); + return c; + } + + template + static Ch Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + return is.Take(); + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(0xEFu); os.Put(0xBBu); os.Put(0xBFu); + } + + template + static void Put(OutputByteStream& os, Ch c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(c)); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// UTF16 + +//! UTF-16 encoding. +/*! http://en.wikipedia.org/wiki/UTF-16 + http://tools.ietf.org/html/rfc2781 + \tparam CharType Type for storing 16-bit UTF-16 data. Default is wchar_t. C++11 may use char16_t instead. + \note implements Encoding concept + + \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. + For streaming, use UTF16LE and UTF16BE, which handle endianness. +*/ +template +struct UTF16 { + typedef CharType Ch; + RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 2); + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + if (codepoint <= 0xFFFF) { + RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair + os.Put(static_cast(codepoint)); + } + else { + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + unsigned v = codepoint - 0x10000; + os.Put(static_cast((v >> 10) | 0xD800)); + os.Put((v & 0x3FF) | 0xDC00); + } + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); + Ch c = is.Take(); + if (c < 0xD800 || c > 0xDFFF) { + *codepoint = c; + return true; + } + else if (c <= 0xDBFF) { + *codepoint = (c & 0x3FF) << 10; + c = is.Take(); + *codepoint |= (c & 0x3FF); + *codepoint += 0x10000; + return c >= 0xDC00 && c <= 0xDFFF; + } + return false; + } + + template + static bool Validate(InputStream& is, OutputStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + Ch c; + os.Put(c = is.Take()); + if (c < 0xD800 || c > 0xDFFF) + return true; + else if (c <= 0xDBFF) { + os.Put(c = is.Take()); + return c >= 0xDC00 && c <= 0xDFFF; + } + return false; + } +}; + +//! UTF-16 little endian encoding. +template +struct UTF16LE : UTF16 { + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return (unsigned short)c == 0xFEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = (unsigned char)is.Take(); + c |= (unsigned char)is.Take() << 8; + return c; + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(0xFFu); os.Put(0xFEu); + } + + template + static void Put(OutputByteStream& os, CharType c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(c & 0xFFu); + os.Put((c >> 8) & 0xFFu); + } +}; + +//! UTF-16 big endian encoding. +template +struct UTF16BE : UTF16 { + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return (unsigned short)c == 0xFEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = (unsigned char)is.Take() << 8; + c |= (unsigned char)is.Take(); + return c; + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(0xFEu); os.Put(0xFFu); + } + + template + static void Put(OutputByteStream& os, CharType c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put((c >> 8) & 0xFFu); + os.Put(c & 0xFFu); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// UTF32 + +//! UTF-32 encoding. +/*! http://en.wikipedia.org/wiki/UTF-32 + \tparam CharType Type for storing 32-bit UTF-32 data. Default is unsigned. C++11 may use char32_t instead. + \note implements Encoding concept + + \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. + For streaming, use UTF32LE and UTF32BE, which handle endianness. +*/ +template +struct UTF32 { + typedef CharType Ch; + RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 4); + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + os.Put(codepoint); + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); + Ch c = is.Take(); + *codepoint = c; + return c <= 0x10FFFF; + } + + template + static bool Validate(InputStream& is, OutputStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); + Ch c; + os.Put(c = is.Take()); + return c <= 0x10FFFF; + } +}; + +//! UTF-32 little endian enocoding. +template +struct UTF32LE : UTF32 { + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return (unsigned)c == 0x0000FEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = (unsigned char)is.Take(); + c |= (unsigned char)is.Take() << 8; + c |= (unsigned char)is.Take() << 16; + c |= (unsigned char)is.Take() << 24; + return c; + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(0xFFu); os.Put(0xFEu); os.Put(0x00u); os.Put(0x00u); + } + + template + static void Put(OutputByteStream& os, CharType c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(c & 0xFFu); + os.Put((c >> 8) & 0xFFu); + os.Put((c >> 16) & 0xFFu); + os.Put((c >> 24) & 0xFFu); + } +}; + +//! UTF-32 big endian encoding. +template +struct UTF32BE : UTF32 { + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return (unsigned)c == 0x0000FEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = (unsigned char)is.Take() << 24; + c |= (unsigned char)is.Take() << 16; + c |= (unsigned char)is.Take() << 8; + c |= (unsigned char)is.Take(); + return c; + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(0x00u); os.Put(0x00u); os.Put(0xFEu); os.Put(0xFFu); + } + + template + static void Put(OutputByteStream& os, CharType c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put((c >> 24) & 0xFFu); + os.Put((c >> 16) & 0xFFu); + os.Put((c >> 8) & 0xFFu); + os.Put(c & 0xFFu); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// ASCII + +//! ASCII encoding. +/*! http://en.wikipedia.org/wiki/ASCII + \tparam CharType Code unit for storing 7-bit ASCII data. Default is char. + \note implements Encoding concept +*/ +template +struct ASCII { + typedef CharType Ch; + + enum { supportUnicode = 0 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + RAPIDJSON_ASSERT(codepoint <= 0x7F); + os.Put(static_cast(codepoint & 0xFF)); + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { + unsigned char c = static_cast(is.Take()); + *codepoint = c; + return c <= 0X7F; + } + + template + static bool Validate(InputStream& is, OutputStream& os) { + unsigned char c = is.Take(); + os.Put(c); + return c <= 0x7F; + } + + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + Ch c = Take(is); + return c; + } + + template + static Ch Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + return is.Take(); + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + (void)os; + } + + template + static void Put(OutputByteStream& os, Ch c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(c)); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// AutoUTF + +//! Runtime-specified UTF encoding type of a stream. +enum UTFType { + kUTF8 = 0, //!< UTF-8. + kUTF16LE = 1, //!< UTF-16 little endian. + kUTF16BE = 2, //!< UTF-16 big endian. + kUTF32LE = 3, //!< UTF-32 little endian. + kUTF32BE = 4 //!< UTF-32 big endian. +}; + +//! Dynamically select encoding according to stream's runtime-specified UTF encoding type. +/*! \note This class can be used with AutoUTFInputtStream and AutoUTFOutputStream, which provides GetType(). +*/ +template +struct AutoUTF { + typedef CharType Ch; + + enum { supportUnicode = 1 }; + +#define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x + + template + RAPIDJSON_FORCEINLINE static void Encode(OutputStream& os, unsigned codepoint) { + typedef void (*EncodeFunc)(OutputStream&, unsigned); + static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Encode) }; + (*f[os.GetType()])(os, codepoint); + } + + template + RAPIDJSON_FORCEINLINE static bool Decode(InputStream& is, unsigned* codepoint) { + typedef bool (*DecodeFunc)(InputStream&, unsigned*); + static const DecodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Decode) }; + return (*f[is.GetType()])(is, codepoint); + } + + template + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + typedef bool (*ValidateFunc)(InputStream&, OutputStream&); + static const ValidateFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Validate) }; + return (*f[is.GetType()])(is, os); + } + +#undef RAPIDJSON_ENCODINGS_FUNC +}; + +/////////////////////////////////////////////////////////////////////////////// +// Transcoder + +//! Encoding conversion. +template +struct Transcoder { + //! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream. + template + RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { + unsigned codepoint; + if (!SourceEncoding::Decode(is, &codepoint)) + return false; + TargetEncoding::Encode(os, codepoint); + return true; + } + + //! Validate one Unicode codepoint from an encoded stream. + template + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + return Transcode(is, os); // Since source/target encoding is different, must transcode. + } +}; + +//! Specialization of Transcoder with same source and target encoding. +template +struct Transcoder { + template + RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { + os.Put(is.Take()); // Just copy one code unit. This semantic is different from primary template class. + return true; + } + + template + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + return Encoding::Validate(is, os); // source/target encoding are the same + } +}; + +} // namespace rapidjson + +#if defined(__GNUC__) || defined(_MSV_VER) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_ENCODINGS_H_ diff --git a/tools/optimizer/rapidjson/error/en.h b/tools/optimizer/rapidjson/error/en.h new file mode 100644 index 0000000000000..d153e04fc8411 --- /dev/null +++ b/tools/optimizer/rapidjson/error/en.h @@ -0,0 +1,71 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_ERROR_EN_H__ +#define RAPIDJSON_ERROR_EN_H__ + +#include "error.h" + +namespace rapidjson { + +//! Maps error code of parsing into error message. +/*! + \ingroup RAPIDJSON_ERRORS + \param parseErrorCode Error code obtained in parsing. + \return the error message. + \note User can make a copy of this function for localization. + Using switch-case is safer for future modification of error codes. +*/ +inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErrorCode) { + switch (parseErrorCode) { + case kParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); + + case kParseErrorDocumentEmpty: return RAPIDJSON_ERROR_STRING("The document is empty."); + case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not follow by other values."); + + case kParseErrorValueInvalid: return RAPIDJSON_ERROR_STRING("Invalid value."); + + case kParseErrorObjectMissName: return RAPIDJSON_ERROR_STRING("Missing a name for object member."); + case kParseErrorObjectMissColon: return RAPIDJSON_ERROR_STRING("Missing a colon after a name of object member."); + case kParseErrorObjectMissCommaOrCurlyBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or '}' after an object member."); + + case kParseErrorArrayMissCommaOrSquareBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or ']' after an array element."); + + case kParseErrorStringUnicodeEscapeInvalidHex: return RAPIDJSON_ERROR_STRING("Incorrect hex digit after \\u escape in string."); + case kParseErrorStringUnicodeSurrogateInvalid: return RAPIDJSON_ERROR_STRING("The surrogate pair in string is invalid."); + case kParseErrorStringEscapeInvalid: return RAPIDJSON_ERROR_STRING("Invalid escape character in string."); + case kParseErrorStringMissQuotationMark: return RAPIDJSON_ERROR_STRING("Missing a closing quotation mark in string."); + case kParseErrorStringInvalidEncoding: return RAPIDJSON_ERROR_STRING("Invalid encoding in string."); + + case kParseErrorNumberTooBig: return RAPIDJSON_ERROR_STRING("Number too big to be stored in double."); + case kParseErrorNumberMissFraction: return RAPIDJSON_ERROR_STRING("Miss fraction part in number."); + case kParseErrorNumberMissExponent: return RAPIDJSON_ERROR_STRING("Miss exponent in number."); + + case kParseErrorTermination: return RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error."); + case kParseErrorUnspecificSyntaxError: return RAPIDJSON_ERROR_STRING("Unspecific syntax error."); + + default: + return RAPIDJSON_ERROR_STRING("Unknown error."); + } +} + +} // namespace rapidjson + +#endif // RAPIDJSON_ERROR_EN_H__ diff --git a/tools/optimizer/rapidjson/error/error.h b/tools/optimizer/rapidjson/error/error.h new file mode 100644 index 0000000000000..146604439b16a --- /dev/null +++ b/tools/optimizer/rapidjson/error/error.h @@ -0,0 +1,150 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_ERROR_ERROR_H__ +#define RAPIDJSON_ERROR_ERROR_H__ + +/*! \file error.h */ + +/*! \defgroup RAPIDJSON_ERRORS RapidJSON error handling */ + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ERROR_CHARTYPE + +//! Character type of error messages. +/*! \ingroup RAPIDJSON_ERRORS + The default character type is \c char. + On Windows, user can define this macro as \c TCHAR for supporting both + unicode/non-unicode settings. +*/ +#ifndef RAPIDJSON_ERROR_CHARTYPE +#define RAPIDJSON_ERROR_CHARTYPE char +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ERROR_STRING + +//! Macro for converting string literial to \ref RAPIDJSON_ERROR_CHARTYPE[]. +/*! \ingroup RAPIDJSON_ERRORS + By default this conversion macro does nothing. + On Windows, user can define this macro as \c _T(x) for supporting both + unicode/non-unicode settings. +*/ +#ifndef RAPIDJSON_ERROR_STRING +#define RAPIDJSON_ERROR_STRING(x) x +#endif + +namespace rapidjson { + +/////////////////////////////////////////////////////////////////////////////// +// ParseErrorCode + +//! Error code of parsing. +/*! \ingroup RAPIDJSON_ERRORS + \see GenericReader::Parse, GenericReader::GetParseErrorCode +*/ +enum ParseErrorCode { + kParseErrorNone = 0, //!< No error. + + kParseErrorDocumentEmpty, //!< The document is empty. + kParseErrorDocumentRootNotSingular, //!< The document root must not follow by other values. + + kParseErrorValueInvalid, //!< Invalid value. + + kParseErrorObjectMissName, //!< Missing a name for object member. + kParseErrorObjectMissColon, //!< Missing a colon after a name of object member. + kParseErrorObjectMissCommaOrCurlyBracket, //!< Missing a comma or '}' after an object member. + + kParseErrorArrayMissCommaOrSquareBracket, //!< Missing a comma or ']' after an array element. + + kParseErrorStringUnicodeEscapeInvalidHex, //!< Incorrect hex digit after \\u escape in string. + kParseErrorStringUnicodeSurrogateInvalid, //!< The surrogate pair in string is invalid. + kParseErrorStringEscapeInvalid, //!< Invalid escape character in string. + kParseErrorStringMissQuotationMark, //!< Missing a closing quotation mark in string. + kParseErrorStringInvalidEncoding, //!< Invalid encoding in string. + + kParseErrorNumberTooBig, //!< Number too big to be stored in double. + kParseErrorNumberMissFraction, //!< Miss fraction part in number. + kParseErrorNumberMissExponent, //!< Miss exponent in number. + + kParseErrorTermination, //!< Parsing was terminated. + kParseErrorUnspecificSyntaxError, //!< Unspecific syntax error. +}; + +//! Result of parsing (wraps ParseErrorCode) +/*! + \ingroup RAPIDJSON_ERRORS + \code + Document doc; + ParseResult ok = doc.Parse("[42]"); + if (!ok) { + fprintf(stderr, "JSON parse error: %s (%u)", + GetParseError_En(ok.Code()), ok.Offset()); + exit(EXIT_FAILURE); + } + \endcode + \see GenericReader::Parse, GenericDocument::Parse +*/ +struct ParseResult { + + //! Default constructor, no error. + ParseResult() : code_(kParseErrorNone), offset_(0) {} + //! Constructor to set an error. + ParseResult(ParseErrorCode code, size_t offset) : code_(code), offset_(offset) {} + + //! Get the error code. + ParseErrorCode Code() const { return code_; } + //! Get the error offset, if \ref IsError(), 0 otherwise. + size_t Offset() const { return offset_; } + + //! Conversion to \c bool, returns \c true, iff !\ref IsError(). + operator bool() const { return !IsError(); } + //! Whether the result is an error. + bool IsError() const { return code_ != kParseErrorNone; } + + bool operator==(const ParseResult& that) const { return code_ == that.code_; } + bool operator==(ParseErrorCode code) const { return code_ == code; } + friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; } + + //! Reset error code. + void Clear() { Set(kParseErrorNone); } + //! Update error code and offset. + void Set(ParseErrorCode code, size_t offset = 0) { code_ = code; offset_ = offset; } + +private: + ParseErrorCode code_; + size_t offset_; +}; + +//! Function pointer type of GetParseError(). +/*! \ingroup RAPIDJSON_ERRORS + + This is the prototype for \c GetParseError_X(), where \c X is a locale. + User can dynamically change locale in runtime, e.g.: +\code + GetParseErrorFunc GetParseError = GetParseError_En; // or whatever + const RAPIDJSON_ERROR_CHARTYPE* s = GetParseError(document.GetParseErrorCode()); +\endcode +*/ +typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetParseErrorFunc)(ParseErrorCode); + +} // namespace rapidjson + +#endif // RAPIDJSON_ERROR_ERROR_H__ diff --git a/tools/optimizer/rapidjson/filereadstream.h b/tools/optimizer/rapidjson/filereadstream.h new file mode 100644 index 0000000000000..31c193bf98d07 --- /dev/null +++ b/tools/optimizer/rapidjson/filereadstream.h @@ -0,0 +1,94 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_FILEREADSTREAM_H_ +#define RAPIDJSON_FILEREADSTREAM_H_ + +#include "rapidjson.h" +#include + +namespace rapidjson { + +//! File byte stream for input using fread(). +/*! + \note implements Stream concept +*/ +class FileReadStream { +public: + typedef char Ch; //!< Character type (byte). + + //! Constructor. + /*! + \param fp File pointer opened for read. + \param buffer user-supplied buffer. + \param bufferSize size of buffer in bytes. Must >=4 bytes. + */ + FileReadStream(FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { + RAPIDJSON_ASSERT(fp_ != 0); + RAPIDJSON_ASSERT(bufferSize >= 4); + Read(); + } + + Ch Peek() const { return *current_; } + Ch Take() { Ch c = *current_; Read(); return c; } + size_t Tell() const { return count_ + static_cast(current_ - buffer_); } + + // Not implemented + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const { + return (current_ + 4 <= bufferLast_) ? current_ : 0; + } + +private: + void Read() { + if (current_ < bufferLast_) + ++current_; + else if (!eof_) { + count_ += readCount_; + readCount_ = fread(buffer_, 1, bufferSize_, fp_); + bufferLast_ = buffer_ + readCount_ - 1; + current_ = buffer_; + + if (readCount_ < bufferSize_) { + buffer_[readCount_] = '\0'; + ++bufferLast_; + eof_ = true; + } + } + } + + FILE* fp_; + Ch *buffer_; + size_t bufferSize_; + Ch *bufferLast_; + Ch *current_; + size_t readCount_; + size_t count_; //!< Number of characters read + bool eof_; +}; + +} // namespace rapidjson + +#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/tools/optimizer/rapidjson/filestream.h b/tools/optimizer/rapidjson/filestream.h new file mode 100644 index 0000000000000..5fe00362b560a --- /dev/null +++ b/tools/optimizer/rapidjson/filestream.h @@ -0,0 +1,73 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_FILESTREAM_H_ +#define RAPIDJSON_FILESTREAM_H_ + +#include "rapidjson.h" +#include + +namespace rapidjson { + +//! (Deprecated) Wrapper of C file stream for input or output. +/*! + This simple wrapper does not check the validity of the stream. + \note implements Stream concept + \note deprecated: This was only for basic testing in version 0.1, it is found that the performance is very low by using fgetc(). Use FileReadStream instead. +*/ +class FileStream { +public: + typedef char Ch; //!< Character type. Only support char. + + FileStream(FILE* fp) : fp_(fp), current_('\0'), count_(0) { Read(); } + char Peek() const { return current_; } + char Take() { char c = current_; Read(); return c; } + size_t Tell() const { return count_; } + void Put(char c) { fputc(c, fp_); } + void Flush() { fflush(fp_); } + + // Not implemented + char* PutBegin() { return 0; } + size_t PutEnd(char*) { return 0; } + +private: + // Prohibit copy constructor & assignment operator. + FileStream(const FileStream&); + FileStream& operator=(const FileStream&); + + void Read() { + RAPIDJSON_ASSERT(fp_ != 0); + int c = fgetc(fp_); + if (c != EOF) { + current_ = (char)c; + count_++; + } + else if (current_ != '\0') + current_ = '\0'; + } + + FILE* fp_; + char current_; + size_t count_; +}; + +} // namespace rapidjson + +#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/tools/optimizer/rapidjson/filewritestream.h b/tools/optimizer/rapidjson/filewritestream.h new file mode 100644 index 0000000000000..05c5ca09077eb --- /dev/null +++ b/tools/optimizer/rapidjson/filewritestream.h @@ -0,0 +1,97 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_FILEWRITESTREAM_H_ +#define RAPIDJSON_FILEWRITESTREAM_H_ + +#include "rapidjson.h" +#include + +namespace rapidjson { + +//! Wrapper of C file stream for input using fread(). +/*! + \note implements Stream concept +*/ +class FileWriteStream { +public: + typedef char Ch; //!< Character type. Only support char. + + FileWriteStream(FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { + RAPIDJSON_ASSERT(fp_ != 0); + } + + void Put(char c) { + if (current_ >= bufferEnd_) + Flush(); + + *current_++ = c; + } + + void PutN(char c, size_t n) { + size_t avail = static_cast(bufferEnd_ - current_); + while (n > avail) { + std::memset(current_, c, avail); + current_ += avail; + Flush(); + n -= avail; + avail = static_cast(bufferEnd_ - current_); + } + + if (n > 0) { + std::memset(current_, c, n); + current_ += n; + } + } + + void Flush() { + if (current_ != buffer_) { + fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); + current_ = buffer_; + } + } + + // Not implemented + char Peek() const { RAPIDJSON_ASSERT(false); return 0; } + char Take() { RAPIDJSON_ASSERT(false); return 0; } + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + // Prohibit copy constructor & assignment operator. + FileWriteStream(const FileWriteStream&); + FileWriteStream& operator=(const FileWriteStream&); + + FILE* fp_; + char *buffer_; + char *bufferEnd_; + char *current_; +}; + +//! Implement specialized version of PutN() with memset() for better performance. +template<> +inline void PutN(FileWriteStream& stream, char c, size_t n) { + stream.PutN(c, n); +} + +} // namespace rapidjson + +#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/tools/optimizer/rapidjson/internal/dtoa.h b/tools/optimizer/rapidjson/internal/dtoa.h new file mode 100644 index 0000000000000..6ae588ac4aa3c --- /dev/null +++ b/tools/optimizer/rapidjson/internal/dtoa.h @@ -0,0 +1,418 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// This is a C++ header-only implementation of Grisu2 algorithm from the publication: +// Loitsch, Florian. "Printing floating-point numbers quickly and accurately with +// integers." ACM Sigplan Notices 45.6 (2010): 233-243. + +#ifndef RAPIDJSON_DTOA_ +#define RAPIDJSON_DTOA_ + +#if defined(_MSC_VER) +#include +#if defined(_M_AMD64) +#pragma intrinsic(_BitScanReverse64) +#endif +#endif + +#include "itoa.h" // GetDigitsLut() + +namespace rapidjson { +namespace internal { + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +struct DiyFp { + DiyFp() {} + + DiyFp(uint64_t f, int e) : f(f), e(e) {} + + DiyFp(double d) { + union { + double d; + uint64_t u64; + } u = { d }; + + int biased_e = static_cast((u.u64 & kDpExponentMask) >> kDpSignificandSize); + uint64_t significand = (u.u64 & kDpSignificandMask); + if (biased_e != 0) { + f = significand + kDpHiddenBit; + e = biased_e - kDpExponentBias; + } + else { + f = significand; + e = kDpMinExponent + 1; + } + } + + DiyFp operator-(const DiyFp& rhs) const { + return DiyFp(f - rhs.f, e); + } + + DiyFp operator*(const DiyFp& rhs) const { +#if defined(_MSC_VER) && defined(_M_AMD64) + uint64_t h; + uint64_t l = _umul128(f, rhs.f, &h); + if (l & (uint64_t(1) << 63)) // rounding + h++; + return DiyFp(h, e + rhs.e + 64); +#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) + unsigned __int128 p = static_cast(f) * static_cast(rhs.f); + uint64_t h = static_cast(p >> 64); + uint64_t l = static_cast(p); + if (l & (uint64_t(1) << 63)) // rounding + h++; + return DiyFp(h, e + rhs.e + 64); +#else + const uint64_t M32 = 0xFFFFFFFF; + const uint64_t a = f >> 32; + const uint64_t b = f & M32; + const uint64_t c = rhs.f >> 32; + const uint64_t d = rhs.f & M32; + const uint64_t ac = a * c; + const uint64_t bc = b * c; + const uint64_t ad = a * d; + const uint64_t bd = b * d; + uint64_t tmp = (bd >> 32) + (ad & M32) + (bc & M32); + tmp += 1U << 31; /// mult_round + return DiyFp(ac + (ad >> 32) + (bc >> 32) + (tmp >> 32), e + rhs.e + 64); +#endif + } + + DiyFp Normalize() const { +#if defined(_MSC_VER) && defined(_M_AMD64) + unsigned long index; + _BitScanReverse64(&index, f); + return DiyFp(f << (63 - index), e - (63 - index)); +#elif defined(__GNUC__) + int s = __builtin_clzll(f); + return DiyFp(f << s, e - s); +#else + DiyFp res = *this; + while (!(res.f & kDpHiddenBit)) { + res.f <<= 1; + res.e--; + } + res.f <<= (kDiySignificandSize - kDpSignificandSize - 1); + res.e = res.e - (kDiySignificandSize - kDpSignificandSize - 1); + return res; +#endif + } + + DiyFp NormalizeBoundary() const { +#if defined(_MSC_VER) && defined(_M_AMD64) + unsigned long index; + _BitScanReverse64(&index, f); + return DiyFp (f << (63 - index), e - (63 - index)); +#else + DiyFp res = *this; + while (!(res.f & (kDpHiddenBit << 1))) { + res.f <<= 1; + res.e--; + } + res.f <<= (kDiySignificandSize - kDpSignificandSize - 2); + res.e = res.e - (kDiySignificandSize - kDpSignificandSize - 2); + return res; +#endif + } + + void NormalizedBoundaries(DiyFp* minus, DiyFp* plus) const { + DiyFp pl = DiyFp((f << 1) + 1, e - 1).NormalizeBoundary(); + DiyFp mi = (f == kDpHiddenBit) ? DiyFp((f << 2) - 1, e - 2) : DiyFp((f << 1) - 1, e - 1); + mi.f <<= mi.e - pl.e; + mi.e = pl.e; + *plus = pl; + *minus = mi; + } + + static const int kDiySignificandSize = 64; + static const int kDpSignificandSize = 52; + static const int kDpExponentBias = 0x3FF + kDpSignificandSize; + static const int kDpMinExponent = -kDpExponentBias; + static const uint64_t kDpExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); + static const uint64_t kDpSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); + static const uint64_t kDpHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); + + uint64_t f; + int e; +}; + +inline DiyFp GetCachedPower(int e, int* K) { + // 10^-348, 10^-340, ..., 10^340 + static const uint64_t kCachedPowers_F[] = { + RAPIDJSON_UINT64_C2(0xfa8fd5a0, 0x081c0288), RAPIDJSON_UINT64_C2(0xbaaee17f, 0xa23ebf76), + RAPIDJSON_UINT64_C2(0x8b16fb20, 0x3055ac76), RAPIDJSON_UINT64_C2(0xcf42894a, 0x5dce35ea), + RAPIDJSON_UINT64_C2(0x9a6bb0aa, 0x55653b2d), RAPIDJSON_UINT64_C2(0xe61acf03, 0x3d1a45df), + RAPIDJSON_UINT64_C2(0xab70fe17, 0xc79ac6ca), RAPIDJSON_UINT64_C2(0xff77b1fc, 0xbebcdc4f), + RAPIDJSON_UINT64_C2(0xbe5691ef, 0x416bd60c), RAPIDJSON_UINT64_C2(0x8dd01fad, 0x907ffc3c), + RAPIDJSON_UINT64_C2(0xd3515c28, 0x31559a83), RAPIDJSON_UINT64_C2(0x9d71ac8f, 0xada6c9b5), + RAPIDJSON_UINT64_C2(0xea9c2277, 0x23ee8bcb), RAPIDJSON_UINT64_C2(0xaecc4991, 0x4078536d), + RAPIDJSON_UINT64_C2(0x823c1279, 0x5db6ce57), RAPIDJSON_UINT64_C2(0xc2109436, 0x4dfb5637), + RAPIDJSON_UINT64_C2(0x9096ea6f, 0x3848984f), RAPIDJSON_UINT64_C2(0xd77485cb, 0x25823ac7), + RAPIDJSON_UINT64_C2(0xa086cfcd, 0x97bf97f4), RAPIDJSON_UINT64_C2(0xef340a98, 0x172aace5), + RAPIDJSON_UINT64_C2(0xb23867fb, 0x2a35b28e), RAPIDJSON_UINT64_C2(0x84c8d4df, 0xd2c63f3b), + RAPIDJSON_UINT64_C2(0xc5dd4427, 0x1ad3cdba), RAPIDJSON_UINT64_C2(0x936b9fce, 0xbb25c996), + RAPIDJSON_UINT64_C2(0xdbac6c24, 0x7d62a584), RAPIDJSON_UINT64_C2(0xa3ab6658, 0x0d5fdaf6), + RAPIDJSON_UINT64_C2(0xf3e2f893, 0xdec3f126), RAPIDJSON_UINT64_C2(0xb5b5ada8, 0xaaff80b8), + RAPIDJSON_UINT64_C2(0x87625f05, 0x6c7c4a8b), RAPIDJSON_UINT64_C2(0xc9bcff60, 0x34c13053), + RAPIDJSON_UINT64_C2(0x964e858c, 0x91ba2655), RAPIDJSON_UINT64_C2(0xdff97724, 0x70297ebd), + RAPIDJSON_UINT64_C2(0xa6dfbd9f, 0xb8e5b88f), RAPIDJSON_UINT64_C2(0xf8a95fcf, 0x88747d94), + RAPIDJSON_UINT64_C2(0xb9447093, 0x8fa89bcf), RAPIDJSON_UINT64_C2(0x8a08f0f8, 0xbf0f156b), + RAPIDJSON_UINT64_C2(0xcdb02555, 0x653131b6), RAPIDJSON_UINT64_C2(0x993fe2c6, 0xd07b7fac), + RAPIDJSON_UINT64_C2(0xe45c10c4, 0x2a2b3b06), RAPIDJSON_UINT64_C2(0xaa242499, 0x697392d3), + RAPIDJSON_UINT64_C2(0xfd87b5f2, 0x8300ca0e), RAPIDJSON_UINT64_C2(0xbce50864, 0x92111aeb), + RAPIDJSON_UINT64_C2(0x8cbccc09, 0x6f5088cc), RAPIDJSON_UINT64_C2(0xd1b71758, 0xe219652c), + RAPIDJSON_UINT64_C2(0x9c400000, 0x00000000), RAPIDJSON_UINT64_C2(0xe8d4a510, 0x00000000), + RAPIDJSON_UINT64_C2(0xad78ebc5, 0xac620000), RAPIDJSON_UINT64_C2(0x813f3978, 0xf8940984), + RAPIDJSON_UINT64_C2(0xc097ce7b, 0xc90715b3), RAPIDJSON_UINT64_C2(0x8f7e32ce, 0x7bea5c70), + RAPIDJSON_UINT64_C2(0xd5d238a4, 0xabe98068), RAPIDJSON_UINT64_C2(0x9f4f2726, 0x179a2245), + RAPIDJSON_UINT64_C2(0xed63a231, 0xd4c4fb27), RAPIDJSON_UINT64_C2(0xb0de6538, 0x8cc8ada8), + RAPIDJSON_UINT64_C2(0x83c7088e, 0x1aab65db), RAPIDJSON_UINT64_C2(0xc45d1df9, 0x42711d9a), + RAPIDJSON_UINT64_C2(0x924d692c, 0xa61be758), RAPIDJSON_UINT64_C2(0xda01ee64, 0x1a708dea), + RAPIDJSON_UINT64_C2(0xa26da399, 0x9aef774a), RAPIDJSON_UINT64_C2(0xf209787b, 0xb47d6b85), + RAPIDJSON_UINT64_C2(0xb454e4a1, 0x79dd1877), RAPIDJSON_UINT64_C2(0x865b8692, 0x5b9bc5c2), + RAPIDJSON_UINT64_C2(0xc83553c5, 0xc8965d3d), RAPIDJSON_UINT64_C2(0x952ab45c, 0xfa97a0b3), + RAPIDJSON_UINT64_C2(0xde469fbd, 0x99a05fe3), RAPIDJSON_UINT64_C2(0xa59bc234, 0xdb398c25), + RAPIDJSON_UINT64_C2(0xf6c69a72, 0xa3989f5c), RAPIDJSON_UINT64_C2(0xb7dcbf53, 0x54e9bece), + RAPIDJSON_UINT64_C2(0x88fcf317, 0xf22241e2), RAPIDJSON_UINT64_C2(0xcc20ce9b, 0xd35c78a5), + RAPIDJSON_UINT64_C2(0x98165af3, 0x7b2153df), RAPIDJSON_UINT64_C2(0xe2a0b5dc, 0x971f303a), + RAPIDJSON_UINT64_C2(0xa8d9d153, 0x5ce3b396), RAPIDJSON_UINT64_C2(0xfb9b7cd9, 0xa4a7443c), + RAPIDJSON_UINT64_C2(0xbb764c4c, 0xa7a44410), RAPIDJSON_UINT64_C2(0x8bab8eef, 0xb6409c1a), + RAPIDJSON_UINT64_C2(0xd01fef10, 0xa657842c), RAPIDJSON_UINT64_C2(0x9b10a4e5, 0xe9913129), + RAPIDJSON_UINT64_C2(0xe7109bfb, 0xa19c0c9d), RAPIDJSON_UINT64_C2(0xac2820d9, 0x623bf429), + RAPIDJSON_UINT64_C2(0x80444b5e, 0x7aa7cf85), RAPIDJSON_UINT64_C2(0xbf21e440, 0x03acdd2d), + RAPIDJSON_UINT64_C2(0x8e679c2f, 0x5e44ff8f), RAPIDJSON_UINT64_C2(0xd433179d, 0x9c8cb841), + RAPIDJSON_UINT64_C2(0x9e19db92, 0xb4e31ba9), RAPIDJSON_UINT64_C2(0xeb96bf6e, 0xbadf77d9), + RAPIDJSON_UINT64_C2(0xaf87023b, 0x9bf0ee6b) + }; + static const int16_t kCachedPowers_E[] = { + -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, + -954, -927, -901, -874, -847, -821, -794, -768, -741, -715, + -688, -661, -635, -608, -582, -555, -529, -502, -475, -449, + -422, -396, -369, -343, -316, -289, -263, -236, -210, -183, + -157, -130, -103, -77, -50, -24, 3, 30, 56, 83, + 109, 136, 162, 189, 216, 242, 269, 295, 322, 348, + 375, 402, 428, 455, 481, 508, 534, 561, 588, 614, + 641, 667, 694, 720, 747, 774, 800, 827, 853, 880, + 907, 933, 960, 986, 1013, 1039, 1066 + }; + + //int k = static_cast(ceil((-61 - e) * 0.30102999566398114)) + 374; + double dk = (-61 - e) * 0.30102999566398114 + 347; // dk must be positive, so can do ceiling in positive + int k = static_cast(dk); + if (k != dk) + k++; + + unsigned index = static_cast((k >> 3) + 1); + *K = -(-348 + static_cast(index << 3)); // decimal exponent no need lookup table + + return DiyFp(kCachedPowers_F[index], kCachedPowers_E[index]); +} + +inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t wp_w) { + while (rest < wp_w && delta - rest >= ten_kappa && + (rest + ten_kappa < wp_w || /// closer + wp_w - rest > rest + ten_kappa - wp_w)) { + buffer[len - 1]--; + rest += ten_kappa; + } +} + +inline unsigned CountDecimalDigit32(uint32_t n) { + // Simple pure C++ implementation was faster than __builtin_clz version in this situation. + if (n < 10) return 1; + if (n < 100) return 2; + if (n < 1000) return 3; + if (n < 10000) return 4; + if (n < 100000) return 5; + if (n < 1000000) return 6; + if (n < 10000000) return 7; + if (n < 100000000) return 8; + if (n < 1000000000) return 9; + return 10; +} + +inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buffer, int* len, int* K) { + static const uint32_t kPow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; + const DiyFp one(uint64_t(1) << -Mp.e, Mp.e); + const DiyFp wp_w = Mp - W; + uint32_t p1 = static_cast(Mp.f >> -one.e); + uint64_t p2 = Mp.f & (one.f - 1); + int kappa = CountDecimalDigit32(p1); + *len = 0; + + while (kappa > 0) { + uint32_t d; + switch (kappa) { + case 10: d = p1 / 1000000000; p1 %= 1000000000; break; + case 9: d = p1 / 100000000; p1 %= 100000000; break; + case 8: d = p1 / 10000000; p1 %= 10000000; break; + case 7: d = p1 / 1000000; p1 %= 1000000; break; + case 6: d = p1 / 100000; p1 %= 100000; break; + case 5: d = p1 / 10000; p1 %= 10000; break; + case 4: d = p1 / 1000; p1 %= 1000; break; + case 3: d = p1 / 100; p1 %= 100; break; + case 2: d = p1 / 10; p1 %= 10; break; + case 1: d = p1; p1 = 0; break; + default: +#if defined(_MSC_VER) + __assume(0); +#elif __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5) + __builtin_unreachable(); +#else + d = 0; +#endif + } + if (d || *len) + buffer[(*len)++] = static_cast('0' + static_cast(d)); + kappa--; + uint64_t tmp = (static_cast(p1) << -one.e) + p2; + if (tmp <= delta) { + *K += kappa; + GrisuRound(buffer, *len, delta, tmp, static_cast(kPow10[kappa]) << -one.e, wp_w.f); + return; + } + } + + // kappa = 0 + for (;;) { + p2 *= 10; + delta *= 10; + char d = static_cast(p2 >> -one.e); + if (d || *len) + buffer[(*len)++] = static_cast('0' + d); + p2 &= one.f - 1; + kappa--; + if (p2 < delta) { + *K += kappa; + GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * kPow10[-kappa]); + return; + } + } +} + +inline void Grisu2(double value, char* buffer, int* length, int* K) { + const DiyFp v(value); + DiyFp w_m, w_p; + v.NormalizedBoundaries(&w_m, &w_p); + + const DiyFp c_mk = GetCachedPower(w_p.e, K); + const DiyFp W = v.Normalize() * c_mk; + DiyFp Wp = w_p * c_mk; + DiyFp Wm = w_m * c_mk; + Wm.f++; + Wp.f--; + DigitGen(W, Wp, Wp.f - Wm.f, buffer, length, K); +} + +inline char* WriteExponent(int K, char* buffer) { + if (K < 0) { + *buffer++ = '-'; + K = -K; + } + + if (K >= 100) { + *buffer++ = static_cast('0' + static_cast(K / 100)); + K %= 100; + const char* d = GetDigitsLut() + K * 2; + *buffer++ = d[0]; + *buffer++ = d[1]; + } + else if (K >= 10) { + const char* d = GetDigitsLut() + K * 2; + *buffer++ = d[0]; + *buffer++ = d[1]; + } + else + *buffer++ = static_cast('0' + static_cast(K)); + + return buffer; +} + +inline char* Prettify(char* buffer, int length, int k) { + const int kk = length + k; // 10^(kk-1) <= v < 10^kk + + if (length <= kk && kk <= 21) { + // 1234e7 -> 12340000000 + for (int i = length; i < kk; i++) + buffer[i] = '0'; + buffer[kk] = '.'; + buffer[kk + 1] = '0'; + return &buffer[kk + 2]; + } + else if (0 < kk && kk <= 21) { + // 1234e-2 -> 12.34 + std::memmove(&buffer[kk + 1], &buffer[kk], length - kk); + buffer[kk] = '.'; + return &buffer[length + 1]; + } + else if (-6 < kk && kk <= 0) { + // 1234e-6 -> 0.001234 + const int offset = 2 - kk; + std::memmove(&buffer[offset], &buffer[0], length); + buffer[0] = '0'; + buffer[1] = '.'; + for (int i = 2; i < offset; i++) + buffer[i] = '0'; + return &buffer[length + offset]; + } + else if (length == 1) { + // 1e30 + buffer[1] = 'e'; + return WriteExponent(kk - 1, &buffer[2]); + } + else { + // 1234e30 -> 1.234e33 + std::memmove(&buffer[2], &buffer[1], length - 1); + buffer[1] = '.'; + buffer[length + 1] = 'e'; + return WriteExponent(kk - 1, &buffer[0 + length + 2]); + } +} + +inline char* dtoa(double value, char* buffer) { + if (value == 0) { + buffer[0] = '0'; + buffer[1] = '.'; + buffer[2] = '0'; + return &buffer[3]; + } + else { + if (value < 0) { + *buffer++ = '-'; + value = -value; + } + int length, K; + Grisu2(value, buffer, &length, &K); + return Prettify(buffer, length, K); + } +} + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +} // namespace internal +} // namespace rapidjson + +#endif // RAPIDJSON_DTOA_ diff --git a/tools/optimizer/rapidjson/internal/itoa.h b/tools/optimizer/rapidjson/internal/itoa.h new file mode 100644 index 0000000000000..425e9830c04a6 --- /dev/null +++ b/tools/optimizer/rapidjson/internal/itoa.h @@ -0,0 +1,306 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_ITOA_ +#define RAPIDJSON_ITOA_ + +namespace rapidjson { +namespace internal { + +inline const char* GetDigitsLut() { + static const char cDigitsLut[200] = { + '0','0','0','1','0','2','0','3','0','4','0','5','0','6','0','7','0','8','0','9', + '1','0','1','1','1','2','1','3','1','4','1','5','1','6','1','7','1','8','1','9', + '2','0','2','1','2','2','2','3','2','4','2','5','2','6','2','7','2','8','2','9', + '3','0','3','1','3','2','3','3','3','4','3','5','3','6','3','7','3','8','3','9', + '4','0','4','1','4','2','4','3','4','4','4','5','4','6','4','7','4','8','4','9', + '5','0','5','1','5','2','5','3','5','4','5','5','5','6','5','7','5','8','5','9', + '6','0','6','1','6','2','6','3','6','4','6','5','6','6','6','7','6','8','6','9', + '7','0','7','1','7','2','7','3','7','4','7','5','7','6','7','7','7','8','7','9', + '8','0','8','1','8','2','8','3','8','4','8','5','8','6','8','7','8','8','8','9', + '9','0','9','1','9','2','9','3','9','4','9','5','9','6','9','7','9','8','9','9' + }; + return cDigitsLut; +} + +inline char* u32toa(uint32_t value, char* buffer) { + const char* cDigitsLut = GetDigitsLut(); + + if (value < 10000) { + const uint32_t d1 = (value / 100) << 1; + const uint32_t d2 = (value % 100) << 1; + + if (value >= 1000) + *buffer++ = cDigitsLut[d1]; + if (value >= 100) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 10) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + } + else if (value < 100000000) { + // value = bbbbcccc + const uint32_t b = value / 10000; + const uint32_t c = value % 10000; + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + if (value >= 10000000) + *buffer++ = cDigitsLut[d1]; + if (value >= 1000000) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 100000) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + } + else { + // value = aabbbbcccc in decimal + + const uint32_t a = value / 100000000; // 1 to 42 + value %= 100000000; + + if (a >= 10) { + const unsigned i = a << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + } + else + *buffer++ = static_cast('0' + static_cast(a)); + + const uint32_t b = value / 10000; // 0 to 9999 + const uint32_t c = value % 10000; // 0 to 9999 + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + *buffer++ = cDigitsLut[d1]; + *buffer++ = cDigitsLut[d1 + 1]; + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + } + return buffer; +} + +inline char* i32toa(int32_t value, char* buffer) { + if (value < 0) { + *buffer++ = '-'; + value = -value; + } + + return u32toa(static_cast(value), buffer); +} + +inline char* u64toa(uint64_t value, char* buffer) { + const char* cDigitsLut = GetDigitsLut(); + const uint64_t kTen8 = 100000000; + const uint64_t kTen9 = kTen8 * 10; + const uint64_t kTen10 = kTen8 * 100; + const uint64_t kTen11 = kTen8 * 1000; + const uint64_t kTen12 = kTen8 * 10000; + const uint64_t kTen13 = kTen8 * 100000; + const uint64_t kTen14 = kTen8 * 1000000; + const uint64_t kTen15 = kTen8 * 10000000; + const uint64_t kTen16 = kTen8 * kTen8; + + if (value < kTen8) { + uint32_t v = static_cast(value); + if (v < 10000) { + const uint32_t d1 = (v / 100) << 1; + const uint32_t d2 = (v % 100) << 1; + + if (v >= 1000) + *buffer++ = cDigitsLut[d1]; + if (v >= 100) + *buffer++ = cDigitsLut[d1 + 1]; + if (v >= 10) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + } + else { + // value = bbbbcccc + const uint32_t b = v / 10000; + const uint32_t c = v % 10000; + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + if (value >= 10000000) + *buffer++ = cDigitsLut[d1]; + if (value >= 1000000) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 100000) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + } + } + else if (value < kTen16) { + const uint32_t v0 = static_cast(value / kTen8); + const uint32_t v1 = static_cast(value % kTen8); + + const uint32_t b0 = v0 / 10000; + const uint32_t c0 = v0 % 10000; + + const uint32_t d1 = (b0 / 100) << 1; + const uint32_t d2 = (b0 % 100) << 1; + + const uint32_t d3 = (c0 / 100) << 1; + const uint32_t d4 = (c0 % 100) << 1; + + const uint32_t b1 = v1 / 10000; + const uint32_t c1 = v1 % 10000; + + const uint32_t d5 = (b1 / 100) << 1; + const uint32_t d6 = (b1 % 100) << 1; + + const uint32_t d7 = (c1 / 100) << 1; + const uint32_t d8 = (c1 % 100) << 1; + + if (value >= kTen15) + *buffer++ = cDigitsLut[d1]; + if (value >= kTen14) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= kTen13) + *buffer++ = cDigitsLut[d2]; + if (value >= kTen12) + *buffer++ = cDigitsLut[d2 + 1]; + if (value >= kTen11) + *buffer++ = cDigitsLut[d3]; + if (value >= kTen10) + *buffer++ = cDigitsLut[d3 + 1]; + if (value >= kTen9) + *buffer++ = cDigitsLut[d4]; + if (value >= kTen8) + *buffer++ = cDigitsLut[d4 + 1]; + + *buffer++ = cDigitsLut[d5]; + *buffer++ = cDigitsLut[d5 + 1]; + *buffer++ = cDigitsLut[d6]; + *buffer++ = cDigitsLut[d6 + 1]; + *buffer++ = cDigitsLut[d7]; + *buffer++ = cDigitsLut[d7 + 1]; + *buffer++ = cDigitsLut[d8]; + *buffer++ = cDigitsLut[d8 + 1]; + } + else { + const uint32_t a = static_cast(value / kTen16); // 1 to 1844 + value %= kTen16; + + if (a < 10) + *buffer++ = static_cast('0' + static_cast(a)); + else if (a < 100) { + const uint32_t i = a << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + } + else if (a < 1000) { + *buffer++ = static_cast('0' + static_cast(a / 100)); + + const uint32_t i = (a % 100) << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + } + else { + const uint32_t i = (a / 100) << 1; + const uint32_t j = (a % 100) << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + *buffer++ = cDigitsLut[j]; + *buffer++ = cDigitsLut[j + 1]; + } + + const uint32_t v0 = static_cast(value / kTen8); + const uint32_t v1 = static_cast(value % kTen8); + + const uint32_t b0 = v0 / 10000; + const uint32_t c0 = v0 % 10000; + + const uint32_t d1 = (b0 / 100) << 1; + const uint32_t d2 = (b0 % 100) << 1; + + const uint32_t d3 = (c0 / 100) << 1; + const uint32_t d4 = (c0 % 100) << 1; + + const uint32_t b1 = v1 / 10000; + const uint32_t c1 = v1 % 10000; + + const uint32_t d5 = (b1 / 100) << 1; + const uint32_t d6 = (b1 % 100) << 1; + + const uint32_t d7 = (c1 / 100) << 1; + const uint32_t d8 = (c1 % 100) << 1; + + *buffer++ = cDigitsLut[d1]; + *buffer++ = cDigitsLut[d1 + 1]; + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + *buffer++ = cDigitsLut[d5]; + *buffer++ = cDigitsLut[d5 + 1]; + *buffer++ = cDigitsLut[d6]; + *buffer++ = cDigitsLut[d6 + 1]; + *buffer++ = cDigitsLut[d7]; + *buffer++ = cDigitsLut[d7 + 1]; + *buffer++ = cDigitsLut[d8]; + *buffer++ = cDigitsLut[d8 + 1]; + } + + return buffer; +} + +inline char* i64toa(int64_t value, char* buffer) { + if (value < 0) { + *buffer++ = '-'; + value = -value; + } + + return u64toa(static_cast(value), buffer); +} + +} // namespace internal +} // namespace rapidjson + +#endif // RAPIDJSON_ITOA_ diff --git a/tools/optimizer/rapidjson/internal/meta.h b/tools/optimizer/rapidjson/internal/meta.h new file mode 100644 index 0000000000000..dbe5450d6b9be --- /dev/null +++ b/tools/optimizer/rapidjson/internal/meta.h @@ -0,0 +1,189 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_INTERNAL_META_H_ +#define RAPIDJSON_INTERNAL_META_H_ + +#ifndef RAPIDJSON_RAPIDJSON_H_ +#error not yet included. Do not include this file directly. +#endif + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif +#if defined(_MSC_VER) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(6334) +#endif + +#if RAPIDJSON_HAS_CXX11_TYPETRAITS +#include +#endif + +//@cond RAPIDJSON_INTERNAL +namespace rapidjson { +namespace internal { + +// Helper to wrap/convert arbitrary types to void, useful for arbitrary type matching +template struct Void { typedef void Type; }; + +/////////////////////////////////////////////////////////////////////////////// +// BoolType, TrueType, FalseType +// +template struct BoolType { + static const bool Value = Cond; + typedef BoolType Type; +}; +typedef BoolType TrueType; +typedef BoolType FalseType; + + +/////////////////////////////////////////////////////////////////////////////// +// SelectIf, BoolExpr, NotExpr, AndExpr, OrExpr +// + +template struct SelectIfImpl { template struct Apply { typedef T1 Type; }; }; +template <> struct SelectIfImpl { template struct Apply { typedef T2 Type; }; }; +template struct SelectIfCond : SelectIfImpl::template Apply {}; +template struct SelectIf : SelectIfCond {}; + +template struct AndExprCond : FalseType {}; +template <> struct AndExprCond : TrueType {}; +template struct OrExprCond : TrueType {}; +template <> struct OrExprCond : FalseType {}; + +template struct BoolExpr : SelectIf::Type {}; +template struct NotExpr : SelectIf::Type {}; +template struct AndExpr : AndExprCond::Type {}; +template struct OrExpr : OrExprCond::Type {}; + + +/////////////////////////////////////////////////////////////////////////////// +// AddConst, MaybeAddConst, RemoveConst +template struct AddConst { typedef const T Type; }; +template struct MaybeAddConst : SelectIfCond {}; +template struct RemoveConst { typedef T Type; }; +template struct RemoveConst { typedef T Type; }; + + +/////////////////////////////////////////////////////////////////////////////// +// IsSame, IsConst, IsMoreConst, IsPointer +// +template struct IsSame : FalseType {}; +template struct IsSame : TrueType {}; + +template struct IsConst : FalseType {}; +template struct IsConst : TrueType {}; + +template +struct IsMoreConst + : AndExpr::Type, typename RemoveConst::Type>, + BoolType::Value >= IsConst::Value> >::Type {}; + +template struct IsPointer : FalseType {}; +template struct IsPointer : TrueType {}; + +/////////////////////////////////////////////////////////////////////////////// +// IsBaseOf +// +#if RAPIDJSON_HAS_CXX11_TYPETRAITS + +template struct IsBaseOf + : BoolType< ::std::is_base_of::value> {}; + +#else // simplified version adopted from Boost + +template struct IsBaseOfImpl { + RAPIDJSON_STATIC_ASSERT(sizeof(B) != 0); + RAPIDJSON_STATIC_ASSERT(sizeof(D) != 0); + + typedef char (&Yes)[1]; + typedef char (&No) [2]; + + template + static Yes Check(const D*, T); + static No Check(const B*, int); + + struct Host { + operator const B*() const; + operator const D*(); + }; + + enum { Value = (sizeof(Check(Host(), 0)) == sizeof(Yes)) }; +}; + +template struct IsBaseOf + : OrExpr, BoolExpr > >::Type {}; + +#endif // RAPIDJSON_HAS_CXX11_TYPETRAITS + + +////////////////////////////////////////////////////////////////////////// +// EnableIf / DisableIf +// +template struct EnableIfCond { typedef T Type; }; +template struct EnableIfCond { /* empty */ }; + +template struct DisableIfCond { typedef T Type; }; +template struct DisableIfCond { /* empty */ }; + +template +struct EnableIf : EnableIfCond {}; + +template +struct DisableIf : DisableIfCond {}; + +// SFINAE helpers +struct SfinaeTag {}; +template struct RemoveSfinaeTag; +template struct RemoveSfinaeTag { typedef T Type; }; + +#define RAPIDJSON_REMOVEFPTR_(type) \ + typename ::rapidjson::internal::RemoveSfinaeTag \ + < ::rapidjson::internal::SfinaeTag&(*) type>::Type + +#define RAPIDJSON_ENABLEIF(cond) \ + typename ::rapidjson::internal::EnableIf \ + ::Type * = NULL + +#define RAPIDJSON_DISABLEIF(cond) \ + typename ::rapidjson::internal::DisableIf \ + ::Type * = NULL + +#define RAPIDJSON_ENABLEIF_RETURN(cond,returntype) \ + typename ::rapidjson::internal::EnableIf \ + ::Type + +#define RAPIDJSON_DISABLEIF_RETURN(cond,returntype) \ + typename ::rapidjson::internal::DisableIf \ + ::Type + +} // namespace internal +} // namespace rapidjson +//@endcond + +#if defined(__GNUC__) || defined(_MSC_VER) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_INTERNAL_META_H_ diff --git a/tools/optimizer/rapidjson/internal/pow10.h b/tools/optimizer/rapidjson/internal/pow10.h new file mode 100644 index 0000000000000..72e0dac9f6bb2 --- /dev/null +++ b/tools/optimizer/rapidjson/internal/pow10.h @@ -0,0 +1,59 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_POW10_ +#define RAPIDJSON_POW10_ + +namespace rapidjson { +namespace internal { + +//! Computes integer powers of 10 in double (10.0^n). +/*! This function uses lookup table for fast and accurate results. + \param n non-negative exponent. Must <= 308. + \return 10.0^n +*/ +inline double Pow10(int n) { + static const double e[] = { // 1e-0...1e308: 309 * 8 bytes = 2472 bytes + 1e+0, + 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, + 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, + 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, + 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, + 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, + 1e+101,1e+102,1e+103,1e+104,1e+105,1e+106,1e+107,1e+108,1e+109,1e+110,1e+111,1e+112,1e+113,1e+114,1e+115,1e+116,1e+117,1e+118,1e+119,1e+120, + 1e+121,1e+122,1e+123,1e+124,1e+125,1e+126,1e+127,1e+128,1e+129,1e+130,1e+131,1e+132,1e+133,1e+134,1e+135,1e+136,1e+137,1e+138,1e+139,1e+140, + 1e+141,1e+142,1e+143,1e+144,1e+145,1e+146,1e+147,1e+148,1e+149,1e+150,1e+151,1e+152,1e+153,1e+154,1e+155,1e+156,1e+157,1e+158,1e+159,1e+160, + 1e+161,1e+162,1e+163,1e+164,1e+165,1e+166,1e+167,1e+168,1e+169,1e+170,1e+171,1e+172,1e+173,1e+174,1e+175,1e+176,1e+177,1e+178,1e+179,1e+180, + 1e+181,1e+182,1e+183,1e+184,1e+185,1e+186,1e+187,1e+188,1e+189,1e+190,1e+191,1e+192,1e+193,1e+194,1e+195,1e+196,1e+197,1e+198,1e+199,1e+200, + 1e+201,1e+202,1e+203,1e+204,1e+205,1e+206,1e+207,1e+208,1e+209,1e+210,1e+211,1e+212,1e+213,1e+214,1e+215,1e+216,1e+217,1e+218,1e+219,1e+220, + 1e+221,1e+222,1e+223,1e+224,1e+225,1e+226,1e+227,1e+228,1e+229,1e+230,1e+231,1e+232,1e+233,1e+234,1e+235,1e+236,1e+237,1e+238,1e+239,1e+240, + 1e+241,1e+242,1e+243,1e+244,1e+245,1e+246,1e+247,1e+248,1e+249,1e+250,1e+251,1e+252,1e+253,1e+254,1e+255,1e+256,1e+257,1e+258,1e+259,1e+260, + 1e+261,1e+262,1e+263,1e+264,1e+265,1e+266,1e+267,1e+268,1e+269,1e+270,1e+271,1e+272,1e+273,1e+274,1e+275,1e+276,1e+277,1e+278,1e+279,1e+280, + 1e+281,1e+282,1e+283,1e+284,1e+285,1e+286,1e+287,1e+288,1e+289,1e+290,1e+291,1e+292,1e+293,1e+294,1e+295,1e+296,1e+297,1e+298,1e+299,1e+300, + 1e+301,1e+302,1e+303,1e+304,1e+305,1e+306,1e+307,1e+308 + }; + RAPIDJSON_ASSERT(n >= 0 && n <= 308); + return e[n]; +} + +} // namespace internal +} // namespace rapidjson + +#endif // RAPIDJSON_POW10_ diff --git a/tools/optimizer/rapidjson/internal/stack.h b/tools/optimizer/rapidjson/internal/stack.h new file mode 100644 index 0000000000000..ea722c6f1918a --- /dev/null +++ b/tools/optimizer/rapidjson/internal/stack.h @@ -0,0 +1,183 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_INTERNAL_STACK_H_ +#define RAPIDJSON_INTERNAL_STACK_H_ + +namespace rapidjson { +namespace internal { + +/////////////////////////////////////////////////////////////////////////////// +// Stack + +//! A type-unsafe stack for storing different types of data. +/*! \tparam Allocator Allocator for allocating stack memory. +*/ +template +class Stack { +public: + // Optimization note: Do not allocate memory for stack_ in constructor. + // Do it lazily when first Push() -> Expand() -> Resize(). + Stack(Allocator* allocator, size_t stackCapacity) : allocator_(allocator), ownAllocator(0), stack_(0), stackTop_(0), stackEnd_(0), initialCapacity_(stackCapacity) { + RAPIDJSON_ASSERT(stackCapacity > 0); + if (!allocator_) + ownAllocator = allocator_ = new Allocator(); + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + Stack(Stack&& rhs) + : allocator_(rhs.allocator_), + ownAllocator(rhs.ownAllocator), + stack_(rhs.stack_), + stackTop_(rhs.stackTop_), + stackEnd_(rhs.stackEnd_), + initialCapacity_(rhs.initialCapacity_) + { + rhs.allocator_ = 0; + rhs.ownAllocator = 0; + rhs.stack_ = 0; + rhs.stackTop_ = 0; + rhs.stackEnd_ = 0; + rhs.initialCapacity_ = 0; + } +#endif + + ~Stack() { + Destroy(); + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + Stack& operator=(Stack&& rhs) { + if (&rhs != this) + { + Destroy(); + + allocator_ = rhs.allocator_; + ownAllocator = rhs.ownAllocator; + stack_ = rhs.stack_; + stackTop_ = rhs.stackTop_; + stackEnd_ = rhs.stackEnd_; + initialCapacity_ = rhs.initialCapacity_; + + rhs.allocator_ = 0; + rhs.ownAllocator = 0; + rhs.stack_ = 0; + rhs.stackTop_ = 0; + rhs.stackEnd_ = 0; + rhs.initialCapacity_ = 0; + } + return *this; + } +#endif + + void Clear() { stackTop_ = stack_; } + + void ShrinkToFit() { + if (Empty()) { + // If the stack is empty, completely deallocate the memory. + Allocator::Free(stack_); + stack_ = 0; + stackTop_ = 0; + stackEnd_ = 0; + } + else + Resize(GetSize()); + } + + // Optimization note: try to minimize the size of this function for force inline. + // Expansion is run very infrequently, so it is moved to another (probably non-inline) function. + template + RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) { + // Expand the stack if needed + if (stackTop_ + sizeof(T) * count >= stackEnd_) + Expand(count); + + T* ret = reinterpret_cast(stackTop_); + stackTop_ += sizeof(T) * count; + return ret; + } + + template + T* Pop(size_t count) { + RAPIDJSON_ASSERT(GetSize() >= count * sizeof(T)); + stackTop_ -= count * sizeof(T); + return reinterpret_cast(stackTop_); + } + + template + T* Top() { + RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); + return reinterpret_cast(stackTop_ - sizeof(T)); + } + + template + T* Bottom() { return (T*)stack_; } + + Allocator& GetAllocator() { return *allocator_; } + bool Empty() const { return stackTop_ == stack_; } + size_t GetSize() const { return static_cast(stackTop_ - stack_); } + size_t GetCapacity() const { return static_cast(stackEnd_ - stack_); } + +private: + template + void Expand(size_t count) { + // Only expand the capacity if the current stack exists. Otherwise just create a stack with initial capacity. + size_t newCapacity; + if (stack_ == 0) + newCapacity = initialCapacity_; + else { + newCapacity = GetCapacity(); + newCapacity += (newCapacity + 1) / 2; + } + size_t newSize = GetSize() + sizeof(T) * count; + if (newCapacity < newSize) + newCapacity = newSize; + + Resize(newCapacity); + } + + void Resize(size_t newCapacity) { + const size_t size = GetSize(); // Backup the current size + stack_ = (char*)allocator_->Realloc(stack_, GetCapacity(), newCapacity); + stackTop_ = stack_ + size; + stackEnd_ = stack_ + newCapacity; + } + + void Destroy() { + Allocator::Free(stack_); + delete ownAllocator; // Only delete if it is owned by the stack + } + + // Prohibit copy constructor & assignment operator. + Stack(const Stack&); + Stack& operator=(const Stack&); + + Allocator* allocator_; + Allocator* ownAllocator; + char *stack_; + char *stackTop_; + char *stackEnd_; + size_t initialCapacity_; +}; + +} // namespace internal +} // namespace rapidjson + +#endif // RAPIDJSON_STACK_H_ diff --git a/tools/optimizer/rapidjson/internal/strfunc.h b/tools/optimizer/rapidjson/internal/strfunc.h new file mode 100644 index 0000000000000..80adcb6b730a1 --- /dev/null +++ b/tools/optimizer/rapidjson/internal/strfunc.h @@ -0,0 +1,43 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ +#define RAPIDJSON_INTERNAL_STRFUNC_H_ + +namespace rapidjson { +namespace internal { + +//! Custom strlen() which works on different character types. +/*! \tparam Ch Character type (e.g. char, wchar_t, short) + \param s Null-terminated input string. + \return Number of characters in the string. + \note This has the same semantics as strlen(), the return value is not number of Unicode codepoints. +*/ +template +inline SizeType StrLen(const Ch* s) { + const Ch* p = s; + while (*p) ++p; + return SizeType(p - s); +} + +} // namespace internal +} // namespace rapidjson + +#endif // RAPIDJSON_INTERNAL_STRFUNC_H_ diff --git a/tools/optimizer/rapidjson/memorybuffer.h b/tools/optimizer/rapidjson/memorybuffer.h new file mode 100644 index 0000000000000..ef15c46780e19 --- /dev/null +++ b/tools/optimizer/rapidjson/memorybuffer.h @@ -0,0 +1,76 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_MEMORYBUFFER_H_ +#define RAPIDJSON_MEMORYBUFFER_H_ + +#include "rapidjson.h" +#include "internal/stack.h" + +namespace rapidjson { + +//! Represents an in-memory output byte stream. +/*! + This class is mainly for being wrapped by EncodedOutputStream or AutoUTFOutputStream. + + It is similar to FileWriteBuffer but the destination is an in-memory buffer instead of a file. + + Differences between MemoryBuffer and StringBuffer: + 1. StringBuffer has Encoding but MemoryBuffer is only a byte buffer. + 2. StringBuffer::GetString() returns a null-terminated string. MemoryBuffer::GetBuffer() returns a buffer without terminator. + + \tparam Allocator type for allocating memory buffer. + \note implements Stream concept +*/ +template +struct GenericMemoryBuffer { + typedef char Ch; // byte + + GenericMemoryBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} + + void Put(Ch c) { *stack_.template Push() = c; } + void Flush() {} + + void Clear() { stack_.Clear(); } + void ShrinkToFit() { stack_.ShrinkToFit(); } + Ch* Push(size_t count) { return stack_.template Push(count); } + void Pop(size_t count) { stack_.template Pop(count); } + + const Ch* GetBuffer() const { + return stack_.template Bottom(); + } + + size_t GetSize() const { return stack_.GetSize(); } + + static const size_t kDefaultCapacity = 256; + mutable internal::Stack stack_; +}; + +typedef GenericMemoryBuffer<> MemoryBuffer; + +//! Implement specialized version of PutN() with memset() for better performance. +template<> +inline void PutN(MemoryBuffer& memoryBuffer, char c, size_t n) { + std::memset(memoryBuffer.stack_.Push(n), c, n * sizeof(c)); +} + +} // namespace rapidjson + +#endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/tools/optimizer/rapidjson/memorystream.h b/tools/optimizer/rapidjson/memorystream.h new file mode 100644 index 0000000000000..6ed226e5a9c6f --- /dev/null +++ b/tools/optimizer/rapidjson/memorystream.h @@ -0,0 +1,67 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_MEMORYSTREAM_H_ +#define RAPIDJSON_MEMORYSTREAM_H_ + +#include "rapidjson.h" + +namespace rapidjson { + +//! Represents an in-memory input byte stream. +/*! + This class is mainly for being wrapped by EncodedInputStream or AutoUTFInputStream. + + It is similar to FileReadBuffer but the source is an in-memory buffer instead of a file. + + Differences between MemoryStream and StringStream: + 1. StringStream has encoding but MemoryStream is a byte stream. + 2. MemoryStream needs size of the source buffer and the buffer don't need to be null terminated. StringStream assume null-terminated string as source. + 3. MemoryStream supports Peek4() for encoding detection. StringStream is specified with an encoding so it should not have Peek4(). + \note implements Stream concept +*/ +struct MemoryStream { + typedef char Ch; // byte + + MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} + + Ch Peek() const { return (src_ == end_) ? '\0' : *src_; } + Ch Take() { return (src_ == end_) ? '\0' : *src_++; } + size_t Tell() const { return static_cast(src_ - begin_); } + + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const { + return Tell() + 4 <= size_ ? src_ : 0; + } + + const Ch* src_; //!< Current read position. + const Ch* begin_; //!< Original head of the string. + const Ch* end_; //!< End of stream. + size_t size_; //!< Size of the stream. +}; + +} // namespace rapidjson + +#endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/tools/optimizer/rapidjson/msinttypes/inttypes.h b/tools/optimizer/rapidjson/msinttypes/inttypes.h new file mode 100644 index 0000000000000..ac7e32b6eef30 --- /dev/null +++ b/tools/optimizer/rapidjson/msinttypes/inttypes.h @@ -0,0 +1,306 @@ +// ISO C9x compliant inttypes.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2013 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the product nor the names of its contributors may +// be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_INTTYPES_H_ // [ +#define _MSC_INTTYPES_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +#include "stdint.h" + +// 7.8 Format conversion of integer types + +typedef struct { + intmax_t quot; + intmax_t rem; +} imaxdiv_t; + +// 7.8.1 Macros for format specifiers + +#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198 + +// The fprintf macros for signed integers are: +#define PRId8 "d" +#define PRIi8 "i" +#define PRIdLEAST8 "d" +#define PRIiLEAST8 "i" +#define PRIdFAST8 "d" +#define PRIiFAST8 "i" + +#define PRId16 "hd" +#define PRIi16 "hi" +#define PRIdLEAST16 "hd" +#define PRIiLEAST16 "hi" +#define PRIdFAST16 "hd" +#define PRIiFAST16 "hi" + +#define PRId32 "I32d" +#define PRIi32 "I32i" +#define PRIdLEAST32 "I32d" +#define PRIiLEAST32 "I32i" +#define PRIdFAST32 "I32d" +#define PRIiFAST32 "I32i" + +#define PRId64 "I64d" +#define PRIi64 "I64i" +#define PRIdLEAST64 "I64d" +#define PRIiLEAST64 "I64i" +#define PRIdFAST64 "I64d" +#define PRIiFAST64 "I64i" + +#define PRIdMAX "I64d" +#define PRIiMAX "I64i" + +#define PRIdPTR "Id" +#define PRIiPTR "Ii" + +// The fprintf macros for unsigned integers are: +#define PRIo8 "o" +#define PRIu8 "u" +#define PRIx8 "x" +#define PRIX8 "X" +#define PRIoLEAST8 "o" +#define PRIuLEAST8 "u" +#define PRIxLEAST8 "x" +#define PRIXLEAST8 "X" +#define PRIoFAST8 "o" +#define PRIuFAST8 "u" +#define PRIxFAST8 "x" +#define PRIXFAST8 "X" + +#define PRIo16 "ho" +#define PRIu16 "hu" +#define PRIx16 "hx" +#define PRIX16 "hX" +#define PRIoLEAST16 "ho" +#define PRIuLEAST16 "hu" +#define PRIxLEAST16 "hx" +#define PRIXLEAST16 "hX" +#define PRIoFAST16 "ho" +#define PRIuFAST16 "hu" +#define PRIxFAST16 "hx" +#define PRIXFAST16 "hX" + +#define PRIo32 "I32o" +#define PRIu32 "I32u" +#define PRIx32 "I32x" +#define PRIX32 "I32X" +#define PRIoLEAST32 "I32o" +#define PRIuLEAST32 "I32u" +#define PRIxLEAST32 "I32x" +#define PRIXLEAST32 "I32X" +#define PRIoFAST32 "I32o" +#define PRIuFAST32 "I32u" +#define PRIxFAST32 "I32x" +#define PRIXFAST32 "I32X" + +#define PRIo64 "I64o" +#define PRIu64 "I64u" +#define PRIx64 "I64x" +#define PRIX64 "I64X" +#define PRIoLEAST64 "I64o" +#define PRIuLEAST64 "I64u" +#define PRIxLEAST64 "I64x" +#define PRIXLEAST64 "I64X" +#define PRIoFAST64 "I64o" +#define PRIuFAST64 "I64u" +#define PRIxFAST64 "I64x" +#define PRIXFAST64 "I64X" + +#define PRIoMAX "I64o" +#define PRIuMAX "I64u" +#define PRIxMAX "I64x" +#define PRIXMAX "I64X" + +#define PRIoPTR "Io" +#define PRIuPTR "Iu" +#define PRIxPTR "Ix" +#define PRIXPTR "IX" + +// The fscanf macros for signed integers are: +#define SCNd8 "d" +#define SCNi8 "i" +#define SCNdLEAST8 "d" +#define SCNiLEAST8 "i" +#define SCNdFAST8 "d" +#define SCNiFAST8 "i" + +#define SCNd16 "hd" +#define SCNi16 "hi" +#define SCNdLEAST16 "hd" +#define SCNiLEAST16 "hi" +#define SCNdFAST16 "hd" +#define SCNiFAST16 "hi" + +#define SCNd32 "ld" +#define SCNi32 "li" +#define SCNdLEAST32 "ld" +#define SCNiLEAST32 "li" +#define SCNdFAST32 "ld" +#define SCNiFAST32 "li" + +#define SCNd64 "I64d" +#define SCNi64 "I64i" +#define SCNdLEAST64 "I64d" +#define SCNiLEAST64 "I64i" +#define SCNdFAST64 "I64d" +#define SCNiFAST64 "I64i" + +#define SCNdMAX "I64d" +#define SCNiMAX "I64i" + +#ifdef _WIN64 // [ +# define SCNdPTR "I64d" +# define SCNiPTR "I64i" +#else // _WIN64 ][ +# define SCNdPTR "ld" +# define SCNiPTR "li" +#endif // _WIN64 ] + +// The fscanf macros for unsigned integers are: +#define SCNo8 "o" +#define SCNu8 "u" +#define SCNx8 "x" +#define SCNX8 "X" +#define SCNoLEAST8 "o" +#define SCNuLEAST8 "u" +#define SCNxLEAST8 "x" +#define SCNXLEAST8 "X" +#define SCNoFAST8 "o" +#define SCNuFAST8 "u" +#define SCNxFAST8 "x" +#define SCNXFAST8 "X" + +#define SCNo16 "ho" +#define SCNu16 "hu" +#define SCNx16 "hx" +#define SCNX16 "hX" +#define SCNoLEAST16 "ho" +#define SCNuLEAST16 "hu" +#define SCNxLEAST16 "hx" +#define SCNXLEAST16 "hX" +#define SCNoFAST16 "ho" +#define SCNuFAST16 "hu" +#define SCNxFAST16 "hx" +#define SCNXFAST16 "hX" + +#define SCNo32 "lo" +#define SCNu32 "lu" +#define SCNx32 "lx" +#define SCNX32 "lX" +#define SCNoLEAST32 "lo" +#define SCNuLEAST32 "lu" +#define SCNxLEAST32 "lx" +#define SCNXLEAST32 "lX" +#define SCNoFAST32 "lo" +#define SCNuFAST32 "lu" +#define SCNxFAST32 "lx" +#define SCNXFAST32 "lX" + +#define SCNo64 "I64o" +#define SCNu64 "I64u" +#define SCNx64 "I64x" +#define SCNX64 "I64X" +#define SCNoLEAST64 "I64o" +#define SCNuLEAST64 "I64u" +#define SCNxLEAST64 "I64x" +#define SCNXLEAST64 "I64X" +#define SCNoFAST64 "I64o" +#define SCNuFAST64 "I64u" +#define SCNxFAST64 "I64x" +#define SCNXFAST64 "I64X" + +#define SCNoMAX "I64o" +#define SCNuMAX "I64u" +#define SCNxMAX "I64x" +#define SCNXMAX "I64X" + +#ifdef _WIN64 // [ +# define SCNoPTR "I64o" +# define SCNuPTR "I64u" +# define SCNxPTR "I64x" +# define SCNXPTR "I64X" +#else // _WIN64 ][ +# define SCNoPTR "lo" +# define SCNuPTR "lu" +# define SCNxPTR "lx" +# define SCNXPTR "lX" +#endif // _WIN64 ] + +#endif // __STDC_FORMAT_MACROS ] + +// 7.8.2 Functions for greatest-width integer types + +// 7.8.2.1 The imaxabs function +#define imaxabs _abs64 + +// 7.8.2.2 The imaxdiv function + +// This is modified version of div() function from Microsoft's div.c found +// in %MSVC.NET%\crt\src\div.c +#ifdef STATIC_IMAXDIV // [ +static +#else // STATIC_IMAXDIV ][ +_inline +#endif // STATIC_IMAXDIV ] +imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom) +{ + imaxdiv_t result; + + result.quot = numer / denom; + result.rem = numer % denom; + + if (numer < 0 && result.rem > 0) { + // did division wrong; must fix up + ++result.quot; + result.rem -= denom; + } + + return result; +} + +// 7.8.2.3 The strtoimax and strtoumax functions +#define strtoimax _strtoi64 +#define strtoumax _strtoui64 + +// 7.8.2.4 The wcstoimax and wcstoumax functions +#define wcstoimax _wcstoi64 +#define wcstoumax _wcstoui64 + + +#endif // _MSC_INTTYPES_H_ ] diff --git a/tools/optimizer/rapidjson/msinttypes/stdint.h b/tools/optimizer/rapidjson/msinttypes/stdint.h new file mode 100644 index 0000000000000..bbad95af1599f --- /dev/null +++ b/tools/optimizer/rapidjson/msinttypes/stdint.h @@ -0,0 +1,296 @@ +// ISO C9x compliant stdint.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2013 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the product nor the names of its contributors may +// be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_STDINT_H_ // [ +#define _MSC_STDINT_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +// miloyip: Originally Visual Studio 2010 uses its own stdint.h. However it generates warning with INT64_C(), so change to use this file for vs2010. +#if _MSC_VER >= 1600 // [ +#include + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +#undef INT8_C +#undef INT16_C +#undef INT32_C +#undef INT64_C +#undef UINT8_C +#undef UINT16_C +#undef UINT32_C +#undef UINT64_C + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +// These #ifndef's are needed to prevent collisions with . +// Check out Issue 9 for the details. +#ifndef INTMAX_C // [ +# define INTMAX_C INT64_C +#endif // INTMAX_C ] +#ifndef UINTMAX_C // [ +# define UINTMAX_C UINT64_C +#endif // UINTMAX_C ] + +#endif // __STDC_CONSTANT_MACROS ] + +#else // ] _MSC_VER >= 1700 [ + +#include + +// For Visual Studio 6 in C++ mode and for many Visual Studio versions when +// compiling for ARM we should wrap include with 'extern "C++" {}' +// or compiler give many errors like this: +// error C2733: second C linkage of overloaded function 'wmemchr' not allowed +#ifdef __cplusplus +extern "C" { +#endif +# include +#ifdef __cplusplus +} +#endif + +// Define _W64 macros to mark types changing their size, like intptr_t. +#ifndef _W64 +# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 +# define _W64 __w64 +# else +# define _W64 +# endif +#endif + + +// 7.18.1 Integer types + +// 7.18.1.1 Exact-width integer types + +// Visual Studio 6 and Embedded Visual C++ 4 doesn't +// realize that, e.g. char has the same size as __int8 +// so we give up on __intX for them. +#if (_MSC_VER < 1300) + typedef signed char int8_t; + typedef signed short int16_t; + typedef signed int int32_t; + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; +#else + typedef signed __int8 int8_t; + typedef signed __int16 int16_t; + typedef signed __int32 int32_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; +#endif +typedef signed __int64 int64_t; +typedef unsigned __int64 uint64_t; + + +// 7.18.1.2 Minimum-width integer types +typedef int8_t int_least8_t; +typedef int16_t int_least16_t; +typedef int32_t int_least32_t; +typedef int64_t int_least64_t; +typedef uint8_t uint_least8_t; +typedef uint16_t uint_least16_t; +typedef uint32_t uint_least32_t; +typedef uint64_t uint_least64_t; + +// 7.18.1.3 Fastest minimum-width integer types +typedef int8_t int_fast8_t; +typedef int16_t int_fast16_t; +typedef int32_t int_fast32_t; +typedef int64_t int_fast64_t; +typedef uint8_t uint_fast8_t; +typedef uint16_t uint_fast16_t; +typedef uint32_t uint_fast32_t; +typedef uint64_t uint_fast64_t; + +// 7.18.1.4 Integer types capable of holding object pointers +#ifdef _WIN64 // [ + typedef signed __int64 intptr_t; + typedef unsigned __int64 uintptr_t; +#else // _WIN64 ][ + typedef _W64 signed int intptr_t; + typedef _W64 unsigned int uintptr_t; +#endif // _WIN64 ] + +// 7.18.1.5 Greatest-width integer types +typedef int64_t intmax_t; +typedef uint64_t uintmax_t; + + +// 7.18.2 Limits of specified-width integer types + +#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 + +// 7.18.2.1 Limits of exact-width integer types +#define INT8_MIN ((int8_t)_I8_MIN) +#define INT8_MAX _I8_MAX +#define INT16_MIN ((int16_t)_I16_MIN) +#define INT16_MAX _I16_MAX +#define INT32_MIN ((int32_t)_I32_MIN) +#define INT32_MAX _I32_MAX +#define INT64_MIN ((int64_t)_I64_MIN) +#define INT64_MAX _I64_MAX +#define UINT8_MAX _UI8_MAX +#define UINT16_MAX _UI16_MAX +#define UINT32_MAX _UI32_MAX +#define UINT64_MAX _UI64_MAX + +// 7.18.2.2 Limits of minimum-width integer types +#define INT_LEAST8_MIN INT8_MIN +#define INT_LEAST8_MAX INT8_MAX +#define INT_LEAST16_MIN INT16_MIN +#define INT_LEAST16_MAX INT16_MAX +#define INT_LEAST32_MIN INT32_MIN +#define INT_LEAST32_MAX INT32_MAX +#define INT_LEAST64_MIN INT64_MIN +#define INT_LEAST64_MAX INT64_MAX +#define UINT_LEAST8_MAX UINT8_MAX +#define UINT_LEAST16_MAX UINT16_MAX +#define UINT_LEAST32_MAX UINT32_MAX +#define UINT_LEAST64_MAX UINT64_MAX + +// 7.18.2.3 Limits of fastest minimum-width integer types +#define INT_FAST8_MIN INT8_MIN +#define INT_FAST8_MAX INT8_MAX +#define INT_FAST16_MIN INT16_MIN +#define INT_FAST16_MAX INT16_MAX +#define INT_FAST32_MIN INT32_MIN +#define INT_FAST32_MAX INT32_MAX +#define INT_FAST64_MIN INT64_MIN +#define INT_FAST64_MAX INT64_MAX +#define UINT_FAST8_MAX UINT8_MAX +#define UINT_FAST16_MAX UINT16_MAX +#define UINT_FAST32_MAX UINT32_MAX +#define UINT_FAST64_MAX UINT64_MAX + +// 7.18.2.4 Limits of integer types capable of holding object pointers +#ifdef _WIN64 // [ +# define INTPTR_MIN INT64_MIN +# define INTPTR_MAX INT64_MAX +# define UINTPTR_MAX UINT64_MAX +#else // _WIN64 ][ +# define INTPTR_MIN INT32_MIN +# define INTPTR_MAX INT32_MAX +# define UINTPTR_MAX UINT32_MAX +#endif // _WIN64 ] + +// 7.18.2.5 Limits of greatest-width integer types +#define INTMAX_MIN INT64_MIN +#define INTMAX_MAX INT64_MAX +#define UINTMAX_MAX UINT64_MAX + +// 7.18.3 Limits of other integer types + +#ifdef _WIN64 // [ +# define PTRDIFF_MIN _I64_MIN +# define PTRDIFF_MAX _I64_MAX +#else // _WIN64 ][ +# define PTRDIFF_MIN _I32_MIN +# define PTRDIFF_MAX _I32_MAX +#endif // _WIN64 ] + +#define SIG_ATOMIC_MIN INT_MIN +#define SIG_ATOMIC_MAX INT_MAX + +#ifndef SIZE_MAX // [ +# ifdef _WIN64 // [ +# define SIZE_MAX _UI64_MAX +# else // _WIN64 ][ +# define SIZE_MAX _UI32_MAX +# endif // _WIN64 ] +#endif // SIZE_MAX ] + +// WCHAR_MIN and WCHAR_MAX are also defined in +#ifndef WCHAR_MIN // [ +# define WCHAR_MIN 0 +#endif // WCHAR_MIN ] +#ifndef WCHAR_MAX // [ +# define WCHAR_MAX _UI16_MAX +#endif // WCHAR_MAX ] + +#define WINT_MIN 0 +#define WINT_MAX _UI16_MAX + +#endif // __STDC_LIMIT_MACROS ] + + +// 7.18.4 Limits of other integer types + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +// These #ifndef's are needed to prevent collisions with . +// Check out Issue 9 for the details. +#ifndef INTMAX_C // [ +# define INTMAX_C INT64_C +#endif // INTMAX_C ] +#ifndef UINTMAX_C // [ +# define UINTMAX_C UINT64_C +#endif // UINTMAX_C ] + +#endif // __STDC_CONSTANT_MACROS ] + +#endif // _MSC_VER >= 1600 ] + +#endif // _MSC_STDINT_H_ ] diff --git a/tools/optimizer/rapidjson/prettywriter.h b/tools/optimizer/rapidjson/prettywriter.h new file mode 100644 index 0000000000000..4eac8d76f8309 --- /dev/null +++ b/tools/optimizer/rapidjson/prettywriter.h @@ -0,0 +1,205 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_PRETTYWRITER_H_ +#define RAPIDJSON_PRETTYWRITER_H_ + +#include "writer.h" + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +namespace rapidjson { + +//! Writer with indentation and spacing. +/*! + \tparam OutputStream Type of ouptut os. + \tparam SourceEncoding Encoding of source string. + \tparam TargetEncoding Encoding of output stream. + \tparam StackAllocator Type of allocator for allocating memory of stack. +*/ +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator> +class PrettyWriter : public Writer { +public: + typedef Writer Base; + typedef typename Base::Ch Ch; + + //! Constructor + /*! \param os Output stream. + \param allocator User supplied allocator. If it is null, it will create a private one. + \param levelDepth Initial capacity of stack. + */ + PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} + + //! Set custom indentation. + /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). + \param indentCharCount Number of indent characters for each indentation level. + \note The default indentation is 4 spaces. + */ + PrettyWriter& SetIndent(Ch indentChar, unsigned indentCharCount) { + RAPIDJSON_ASSERT(indentChar == ' ' || indentChar == '\t' || indentChar == '\n' || indentChar == '\r'); + indentChar_ = indentChar; + indentCharCount_ = indentCharCount; + return *this; + } + + /*! @name Implementation of Handler + \see Handler + */ + //@{ + + bool Null() { PrettyPrefix(kNullType); return Base::WriteNull(); } + bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::WriteBool(b); } + bool Int(int i) { PrettyPrefix(kNumberType); return Base::WriteInt(i); } + bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::WriteUint(u); } + bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::WriteInt64(i64); } + bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); } + bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); } + + bool String(const Ch* str, SizeType length, bool copy = false) { + (void)copy; + PrettyPrefix(kStringType); + return Base::WriteString(str, length); + } + + bool StartObject() { + PrettyPrefix(kObjectType); + new (Base::level_stack_.template Push()) typename Base::Level(false); + return Base::WriteStartObject(); + } + + bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + + bool EndObject(SizeType memberCount = 0) { + (void)memberCount; + RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); + RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; + + if (!empty) { + Base::os_->Put('\n'); + WriteIndent(); + } + if (!Base::WriteEndObject()) + return false; + if (Base::level_stack_.Empty()) // end of json text + Base::os_->Flush(); + return true; + } + + bool StartArray() { + PrettyPrefix(kArrayType); + new (Base::level_stack_.template Push()) typename Base::Level(true); + return Base::WriteStartArray(); + } + + bool EndArray(SizeType memberCount = 0) { + (void)memberCount; + RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); + RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; + + if (!empty) { + Base::os_->Put('\n'); + WriteIndent(); + } + if (!Base::WriteEndArray()) + return false; + if (Base::level_stack_.Empty()) // end of json text + Base::os_->Flush(); + return true; + } + + //@} + + /*! @name Convenience extensions */ + //@{ + + //! Simpler but slower overload. + bool String(const Ch* str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } + + //@} +protected: + void PrettyPrefix(Type type) { + (void)type; + if (Base::level_stack_.GetSize() != 0) { // this value is not at root + typename Base::Level* level = Base::level_stack_.template Top(); + + if (level->inArray) { + if (level->valueCount > 0) { + Base::os_->Put(','); // add comma if it is not the first element in array + Base::os_->Put('\n'); + } + else + Base::os_->Put('\n'); + WriteIndent(); + } + else { // in object + if (level->valueCount > 0) { + if (level->valueCount % 2 == 0) { + Base::os_->Put(','); + Base::os_->Put('\n'); + } + else { + Base::os_->Put(':'); + Base::os_->Put(' '); + } + } + else + Base::os_->Put('\n'); + + if (level->valueCount % 2 == 0) + WriteIndent(); + } + if (!level->inArray && level->valueCount % 2 == 0) + RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name + level->valueCount++; + } + else { + RAPIDJSON_ASSERT(!Base::hasRoot_); // Should only has one and only one root. + Base::hasRoot_ = true; + } + } + + void WriteIndent() { + size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; + PutN(*Base::os_, indentChar_, count); + } + + Ch indentChar_; + unsigned indentCharCount_; + +private: + // Prohibit copy constructor & assignment operator. + PrettyWriter(const PrettyWriter&); + PrettyWriter& operator=(const PrettyWriter&); +}; + +} // namespace rapidjson + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/tools/optimizer/rapidjson/rapidjson.h b/tools/optimizer/rapidjson/rapidjson.h new file mode 100644 index 0000000000000..3f743234ddd55 --- /dev/null +++ b/tools/optimizer/rapidjson/rapidjson.h @@ -0,0 +1,567 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_RAPIDJSON_H_ +#define RAPIDJSON_RAPIDJSON_H_ + +// Copyright (c) 2011 Milo Yip (miloyip@gmail.com) +// Version 0.1 + +/*!\file rapidjson.h + \brief common definitions and configuration + + \see RAPIDJSON_CONFIG + */ + +/*! \defgroup RAPIDJSON_CONFIG RapidJSON configuration + \brief Configuration macros for library features + + Some RapidJSON features are configurable to adapt the library to a wide + variety of platforms, environments and usage scenarios. Most of the + features can be configured in terms of overriden or predefined + preprocessor macros at compile-time. + + Some additional customization is available in the \ref RAPIDJSON_ERRORS APIs. + + \note These macros should be given on the compiler command-line + (where applicable) to avoid inconsistent values when compiling + different translation units of a single application. + */ + +#include // malloc(), realloc(), free(), size_t +#include // memset(), memcpy(), memmove(), memcmp() + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_NO_INT64DEFINE + +/*! \def RAPIDJSON_NO_INT64DEFINE + \ingroup RAPIDJSON_CONFIG + \brief Use external 64-bit integer types. + + RapidJSON requires the 64-bit integer types \c int64_t and \c uint64_t types + to be available at global scope. + + If users have their own definition, define RAPIDJSON_NO_INT64DEFINE to + prevent RapidJSON from defining its own types. +*/ +#ifndef RAPIDJSON_NO_INT64DEFINE +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#ifdef _MSC_VER +#include "msinttypes/stdint.h" +#include "msinttypes/inttypes.h" +#else +// Other compilers should have this. +#include +#include +#endif +//!@endcond +#ifdef RAPIDJSON_DOXYGEN_RUNNING +#define RAPIDJSON_NO_INT64DEFINE +#endif +#endif // RAPIDJSON_NO_INT64TYPEDEF + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_FORCEINLINE + +#ifndef RAPIDJSON_FORCEINLINE +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#ifdef _MSC_VER +#define RAPIDJSON_FORCEINLINE __forceinline +#elif defined(__GNUC__) && __GNUC__ >= 4 +#define RAPIDJSON_FORCEINLINE __attribute__((always_inline)) +#else +#define RAPIDJSON_FORCEINLINE +#endif +//!@endcond +#endif // RAPIDJSON_FORCEINLINE + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ENDIAN +#define RAPIDJSON_LITTLEENDIAN 0 //!< Little endian machine +#define RAPIDJSON_BIGENDIAN 1 //!< Big endian machine + +//! Endianness of the machine. +/*! + \def RAPIDJSON_ENDIAN + \ingroup RAPIDJSON_CONFIG + + GCC 4.6 provided macro for detecting endianness of the target machine. But other + compilers may not have this. User can define RAPIDJSON_ENDIAN to either + \ref RAPIDJSON_LITTLEENDIAN or \ref RAPIDJSON_BIGENDIAN. + + Default detection implemented with reference to + \li https://gcc.gnu.org/onlinedocs/gcc-4.6.0/cpp/Common-Predefined-Macros.html + \li http://www.boost.org/doc/libs/1_42_0/boost/detail/endian.hpp +*/ +#ifndef RAPIDJSON_ENDIAN +// Detect with GCC 4.6's macro +# ifdef __BYTE_ORDER__ +# if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN +# else +# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. +# endif // __BYTE_ORDER__ +// Detect with GLIBC's endian.h +# elif defined(__GLIBC__) +# include +# if (__BYTE_ORDER == __LITTLE_ENDIAN) +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif (__BYTE_ORDER == __BIG_ENDIAN) +# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN +# else +# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. +# endif // __GLIBC__ +// Detect with _LITTLE_ENDIAN and _BIG_ENDIAN macro +# elif defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN) +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN) +# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN +// Detect with architecture macros +# elif defined(__sparc) || defined(__sparc__) || defined(_POWER) || defined(__powerpc__) || defined(__ppc__) || defined(__hpux) || defined(__hppa) || defined(_MIPSEB) || defined(_POWER) || defined(__s390__) +# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN +# elif defined(__i386__) || defined(__alpha__) || defined(__ia64) || defined(__ia64__) || defined(_M_IX86) || defined(_M_IA64) || defined(_M_ALPHA) || defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) || defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || defined(__bfin__) +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif defined(RAPIDJSON_DOXYGEN_RUNNING) +# define RAPIDJSON_ENDIAN +# else +# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. +# endif +#endif // RAPIDJSON_ENDIAN + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_64BIT + +//! Whether using 64-bit architecture +#ifndef RAPIDJSON_64BIT +#if defined(__LP64__) || defined(_WIN64) +#define RAPIDJSON_64BIT 1 +#else +#define RAPIDJSON_64BIT 0 +#endif +#endif // RAPIDJSON_64BIT + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ALIGN + +//! Data alignment of the machine. +/*! \ingroup RAPIDJSON_CONFIG + \param x pointer to align + + Some machines require strict data alignment. Currently the default uses 4 bytes + alignment. User can customize by defining the RAPIDJSON_ALIGN function macro., +*/ +#ifndef RAPIDJSON_ALIGN +#define RAPIDJSON_ALIGN(x) ((x + 3u) & ~3u) +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_UINT64_C2 + +//! Construct a 64-bit literal by a pair of 32-bit integer. +/*! + 64-bit literal with or without ULL suffix is prone to compiler warnings. + UINT64_C() is C macro which cause compilation problems. + Use this macro to define 64-bit constants by a pair of 32-bit integer. +*/ +#ifndef RAPIDJSON_UINT64_C2 +#define RAPIDJSON_UINT64_C2(high32, low32) ((static_cast(high32) << 32) | static_cast(low32)) +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_SIMD + +/*! \def RAPIDJSON_SIMD + \ingroup RAPIDJSON_CONFIG + \brief Enable SSE2/SSE4.2 optimization. + + RapidJSON supports optimized implementations for some parsing operations + based on the SSE2 or SSE4.2 SIMD extensions on modern Intel-compatible + processors. + + To enable these optimizations, two different symbols can be defined; + \code + // Enable SSE2 optimization. + #define RAPIDJSON_SSE2 + + // Enable SSE4.2 optimization. + #define RAPIDJSON_SSE42 + \endcode + + \c RAPIDJSON_SSE42 takes precedence, if both are defined. + + If any of these symbols is defined, RapidJSON defines the macro + \c RAPIDJSON_SIMD to indicate the availability of the optimized code. +*/ +#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) \ + || defined(RAPIDJSON_DOXYGEN_RUNNING) +#define RAPIDJSON_SIMD +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_NO_SIZETYPEDEFINE + +#ifndef RAPIDJSON_NO_SIZETYPEDEFINE +/*! \def RAPIDJSON_NO_SIZETYPEDEFINE + \ingroup RAPIDJSON_CONFIG + \brief User-provided \c SizeType definition. + + In order to avoid using 32-bit size types for indexing strings and arrays, + define this preprocessor symbol and provide the type rapidjson::SizeType + before including RapidJSON: + \code + #define RAPIDJSON_NO_SIZETYPEDEFINE + namespace rapidjson { typedef ::std::size_t SizeType; } + #include "rapidjson/..." + \endcode + + \see rapidjson::SizeType +*/ +#ifdef RAPIDJSON_DOXYGEN_RUNNING +#define RAPIDJSON_NO_SIZETYPEDEFINE +#endif +namespace rapidjson { +//! Size type (for string lengths, array sizes, etc.) +/*! RapidJSON uses 32-bit array/string indices even on 64-bit platforms, + instead of using \c size_t. Users may override the SizeType by defining + \ref RAPIDJSON_NO_SIZETYPEDEFINE. +*/ +typedef unsigned SizeType; +} // namespace rapidjson +#endif + +// always import std::size_t to rapidjson namespace +namespace rapidjson { +using std::size_t; +} // namespace rapidjson + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ASSERT + +//! Assertion. +/*! \ingroup RAPIDJSON_CONFIG + By default, rapidjson uses C \c assert() for internal assertions. + User can override it by defining RAPIDJSON_ASSERT(x) macro. + + \note Parsing errors are handled and can be customized by the + \ref RAPIDJSON_ERRORS APIs. +*/ +#ifndef RAPIDJSON_ASSERT +#include +#define RAPIDJSON_ASSERT(x) assert(x) +#endif // RAPIDJSON_ASSERT + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_STATIC_ASSERT + +// Adopt from boost +#ifndef RAPIDJSON_STATIC_ASSERT +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +namespace rapidjson { + +template struct STATIC_ASSERTION_FAILURE; +template <> struct STATIC_ASSERTION_FAILURE { enum { value = 1 }; }; +template struct StaticAssertTest {}; +} // namespace rapidjson + +#define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) +#define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) +#define RAPIDJSON_DO_JOIN2(X, Y) X##Y + +#if defined(__GNUC__) +#define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE __attribute__((unused)) +#else +#define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE +#endif +//!@endcond + +/*! \def RAPIDJSON_STATIC_ASSERT + \brief (Internal) macro to check for conditions at compile-time + \param x compile-time condition + \hideinitializer + */ +#define RAPIDJSON_STATIC_ASSERT(x) typedef ::rapidjson::StaticAssertTest<\ + sizeof(::rapidjson::STATIC_ASSERTION_FAILURE)>\ + RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE +#endif + +/////////////////////////////////////////////////////////////////////////////// +// Helpers + +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN + +#define RAPIDJSON_MULTILINEMACRO_BEGIN do { +#define RAPIDJSON_MULTILINEMACRO_END \ +} while((void)0, 0) + +// adopted from Boost +#define RAPIDJSON_VERSION_CODE(x,y,z) \ + (((x)*100000) + ((y)*100) + (z)) + +// token stringification +#define RAPIDJSON_STRINGIFY(x) RAPIDJSON_DO_STRINGIFY(x) +#define RAPIDJSON_DO_STRINGIFY(x) #x + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_DIAG_PUSH/POP, RAPIDJSON_DIAG_OFF + +#if defined(__GNUC__) +#define RAPIDJSON_GNUC \ + RAPIDJSON_VERSION_CODE(__GNUC__,__GNUC_MINOR__,__GNUC_PATCHLEVEL__) +#endif + +#if defined(__clang__) || (defined(RAPIDJSON_GNUC) && RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,2,0)) + +#define RAPIDJSON_PRAGMA(x) _Pragma(RAPIDJSON_STRINGIFY(x)) +#define RAPIDJSON_DIAG_PRAGMA(x) RAPIDJSON_PRAGMA(GCC diagnostic x) +#define RAPIDJSON_DIAG_OFF(x) \ + RAPIDJSON_DIAG_PRAGMA(ignored RAPIDJSON_STRINGIFY(RAPIDJSON_JOIN(-W,x))) + +// push/pop support in Clang and GCC>=4.6 +#if defined(__clang__) || (defined(RAPIDJSON_GNUC) && RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) +#define RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_PRAGMA(push) +#define RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_PRAGMA(pop) +#else // GCC >= 4.2, < 4.6 +#define RAPIDJSON_DIAG_PUSH /* ignored */ +#define RAPIDJSON_DIAG_POP /* ignored */ +#endif + +#elif defined(_MSC_VER) + +// pragma (MSVC specific) +#define RAPIDJSON_PRAGMA(x) __pragma(x) +#define RAPIDJSON_DIAG_PRAGMA(x) RAPIDJSON_PRAGMA(warning(x)) + +#define RAPIDJSON_DIAG_OFF(x) RAPIDJSON_DIAG_PRAGMA(disable: x) +#define RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_PRAGMA(push) +#define RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_PRAGMA(pop) + +#else + +#define RAPIDJSON_DIAG_OFF(x) /* ignored */ +#define RAPIDJSON_DIAG_PUSH /* ignored */ +#define RAPIDJSON_DIAG_POP /* ignored */ + +#endif // RAPIDJSON_DIAG_* + +/////////////////////////////////////////////////////////////////////////////// +// C++11 features + +#ifndef RAPIDJSON_HAS_CXX11_RVALUE_REFS +#if defined(__clang__) +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS __has_feature(cxx_rvalue_references) +#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ + (defined(_MSC_VER) && _MSC_VER >= 1600) + +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 +#else +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 +#endif +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + +#ifndef RAPIDJSON_HAS_CXX11_NOEXCEPT +#if defined(__clang__) +#define RAPIDJSON_HAS_CXX11_NOEXCEPT __has_feature(cxx_noexcept) +#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) +// (defined(_MSC_VER) && _MSC_VER >= ????) // not yet supported +#define RAPIDJSON_HAS_CXX11_NOEXCEPT 1 +#else +#define RAPIDJSON_HAS_CXX11_NOEXCEPT 0 +#endif +#endif +#if RAPIDJSON_HAS_CXX11_NOEXCEPT +#define RAPIDJSON_NOEXCEPT noexcept +#else +#define RAPIDJSON_NOEXCEPT /* noexcept */ +#endif // RAPIDJSON_HAS_CXX11_NOEXCEPT + +// no automatic detection, yet +#ifndef RAPIDJSON_HAS_CXX11_TYPETRAITS +#define RAPIDJSON_HAS_CXX11_TYPETRAITS 0 +#endif + +//!@endcond + +/////////////////////////////////////////////////////////////////////////////// +// Allocators and Encodings + +#include "allocators.h" +#include "encodings.h" + +//! main RapidJSON namespace +namespace rapidjson { + +/////////////////////////////////////////////////////////////////////////////// +// Stream + +/*! \class rapidjson::Stream + \brief Concept for reading and writing characters. + + For read-only stream, no need to implement PutBegin(), Put(), Flush() and PutEnd(). + + For write-only stream, only need to implement Put() and Flush(). + +\code +concept Stream { + typename Ch; //!< Character type of the stream. + + //! Read the current character from stream without moving the read cursor. + Ch Peek() const; + + //! Read the current character from stream and moving the read cursor to next character. + Ch Take(); + + //! Get the current read cursor. + //! \return Number of characters read from start. + size_t Tell(); + + //! Begin writing operation at the current read pointer. + //! \return The begin writer pointer. + Ch* PutBegin(); + + //! Write a character. + void Put(Ch c); + + //! Flush the buffer. + void Flush(); + + //! End the writing operation. + //! \param begin The begin write pointer returned by PutBegin(). + //! \return Number of characters written. + size_t PutEnd(Ch* begin); +} +\endcode +*/ + +//! Provides additional information for stream. +/*! + By using traits pattern, this type provides a default configuration for stream. + For custom stream, this type can be specialized for other configuration. + See TEST(Reader, CustomStringStream) in readertest.cpp for example. +*/ +template +struct StreamTraits { + //! Whether to make local copy of stream for optimization during parsing. + /*! + By default, for safety, streams do not use local copy optimization. + Stream that can be copied fast should specialize this, like StreamTraits. + */ + enum { copyOptimization = 0 }; +}; + +//! Put N copies of a character to a stream. +template +inline void PutN(Stream& stream, Ch c, size_t n) { + for (size_t i = 0; i < n; i++) + stream.Put(c); +} + +/////////////////////////////////////////////////////////////////////////////// +// StringStream + +//! Read-only string stream. +/*! \note implements Stream concept +*/ +template +struct GenericStringStream { + typedef typename Encoding::Ch Ch; + + GenericStringStream(const Ch *src) : src_(src), head_(src) {} + + Ch Peek() const { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() const { return static_cast(src_ - head_); } + + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + const Ch* src_; //!< Current read position. + const Ch* head_; //!< Original head of the string. +}; + +template +struct StreamTraits > { + enum { copyOptimization = 1 }; +}; + +//! String stream with UTF8 encoding. +typedef GenericStringStream > StringStream; + +/////////////////////////////////////////////////////////////////////////////// +// InsituStringStream + +//! A read-write string stream. +/*! This string stream is particularly designed for in-situ parsing. + \note implements Stream concept +*/ +template +struct GenericInsituStringStream { + typedef typename Encoding::Ch Ch; + + GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} + + // Read + Ch Peek() { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() { return static_cast(src_ - head_); } + + // Write + void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } + + Ch* PutBegin() { return dst_ = src_; } + size_t PutEnd(Ch* begin) { return static_cast(dst_ - begin); } + void Flush() {} + + Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } + void Pop(size_t count) { dst_ -= count; } + + Ch* src_; + Ch* dst_; + Ch* head_; +}; + +template +struct StreamTraits > { + enum { copyOptimization = 1 }; +}; + +//! Insitu string stream with UTF8 encoding. +typedef GenericInsituStringStream > InsituStringStream; + +/////////////////////////////////////////////////////////////////////////////// +// Type + +//! Type of JSON value +enum Type { + kNullType = 0, //!< null + kFalseType = 1, //!< false + kTrueType = 2, //!< true + kObjectType = 3, //!< object + kArrayType = 4, //!< array + kStringType = 5, //!< string + kNumberType = 6 //!< number +}; + +} // namespace rapidjson + +#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/tools/optimizer/rapidjson/reader.h b/tools/optimizer/rapidjson/reader.h new file mode 100644 index 0000000000000..f41ba2fd7b82a --- /dev/null +++ b/tools/optimizer/rapidjson/reader.h @@ -0,0 +1,1369 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_READER_H_ +#define RAPIDJSON_READER_H_ + +/*! \file reader.h */ + +#include "rapidjson.h" +#include "encodings.h" +#include "internal/meta.h" +#include "internal/pow10.h" +#include "internal/stack.h" + +#if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) +#include +#pragma intrinsic(_BitScanForward) +#endif +#ifdef RAPIDJSON_SSE42 +#include +#elif defined(RAPIDJSON_SSE2) +#include +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +RAPIDJSON_DIAG_OFF(4702) // unreachable code +#endif + +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#define RAPIDJSON_NOTHING /* deliberately empty */ +#ifndef RAPIDJSON_PARSE_ERROR_EARLY_RETURN +#define RAPIDJSON_PARSE_ERROR_EARLY_RETURN(value) \ + RAPIDJSON_MULTILINEMACRO_BEGIN \ + if (HasParseError()) { return value; } \ + RAPIDJSON_MULTILINEMACRO_END +#endif +#define RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID \ + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(RAPIDJSON_NOTHING) +//!@endcond + +/*! \def RAPIDJSON_PARSE_ERROR_NORETURN + \ingroup RAPIDJSON_ERRORS + \brief Macro to indicate a parse error. + \param parseErrorCode \ref rapidjson::ParseErrorCode of the error + \param offset position of the error in JSON input (\c size_t) + + This macros can be used as a customization point for the internal + error handling mechanism of RapidJSON. + + A common usage model is to throw an exception instead of requiring the + caller to explicitly check the \ref rapidjson::GenericReader::Parse's + return value: + + \code + #define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode,offset) \ + throw ParseException(parseErrorCode, #parseErrorCode, offset) + + #include // std::runtime_error + #include "rapidjson/error/error.h" // rapidjson::ParseResult + + struct ParseException : std::runtime_error, rapidjson::ParseResult { + ParseException(rapidjson::ParseErrorCode code, const char* msg, size_t offset) + : std::runtime_error(msg), ParseResult(code, offset) {} + }; + + #include "rapidjson/reader.h" + \endcode + + \see RAPIDJSON_PARSE_ERROR, rapidjson::GenericReader::Parse + */ +#ifndef RAPIDJSON_PARSE_ERROR_NORETURN +#define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset) \ + RAPIDJSON_MULTILINEMACRO_BEGIN \ + RAPIDJSON_ASSERT(!HasParseError()); /* Error can only be assigned once */ \ + SetParseError(parseErrorCode, offset); \ + RAPIDJSON_MULTILINEMACRO_END +#endif + +/*! \def RAPIDJSON_PARSE_ERROR + \ingroup RAPIDJSON_ERRORS + \brief (Internal) macro to indicate and handle a parse error. + \param parseErrorCode \ref rapidjson::ParseErrorCode of the error + \param offset position of the error in JSON input (\c size_t) + + Invokes RAPIDJSON_PARSE_ERROR_NORETURN and stops the parsing. + + \see RAPIDJSON_PARSE_ERROR_NORETURN + \hideinitializer + */ +#ifndef RAPIDJSON_PARSE_ERROR +#define RAPIDJSON_PARSE_ERROR(parseErrorCode, offset) \ + RAPIDJSON_MULTILINEMACRO_BEGIN \ + RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset); \ + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; \ + RAPIDJSON_MULTILINEMACRO_END +#endif + +#include "error/error.h" // ParseErrorCode, ParseResult + +namespace rapidjson { + +/////////////////////////////////////////////////////////////////////////////// +// ParseFlag + +//! Combination of parseFlags +/*! \see Reader::Parse, Document::Parse, Document::ParseInsitu, Document::ParseStream + */ +enum ParseFlag { + kParseDefaultFlags = 0, //!< Default parse flags. Non-destructive parsing. Text strings are decoded into allocated buffer. + kParseInsituFlag = 1, //!< In-situ(destructive) parsing. + kParseValidateEncodingFlag = 2, //!< Validate encoding of JSON strings. + kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing. + kParseStopWhenDoneFlag = 8 //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. +}; + +/////////////////////////////////////////////////////////////////////////////// +// Handler + +/*! \class rapidjson::Handler + \brief Concept for receiving events from GenericReader upon parsing. + The functions return true if no error occurs. If they return false, + the event publisher should terminate the process. +\code +concept Handler { + typename Ch; + + bool Null(); + bool Bool(bool b); + bool Int(int i); + bool Uint(unsigned i); + bool Int64(int64_t i); + bool Uint64(uint64_t i); + bool Double(double d); + bool String(const Ch* str, SizeType length, bool copy); + bool StartObject(); + bool Key(const Ch* str, SizeType length, bool copy); + bool EndObject(SizeType memberCount); + bool StartArray(); + bool EndArray(SizeType elementCount); +}; +\endcode +*/ +/////////////////////////////////////////////////////////////////////////////// +// BaseReaderHandler + +//! Default implementation of Handler. +/*! This can be used as base class of any reader handler. + \note implements Handler concept +*/ +template, typename Derived = void> +struct BaseReaderHandler { + typedef typename Encoding::Ch Ch; + + typedef typename internal::SelectIf, BaseReaderHandler, Derived>::Type Override; + + bool Default() { return true; } + bool Null() { return static_cast(*this).Default(); } + bool Bool(bool) { return static_cast(*this).Default(); } + bool Int(int) { return static_cast(*this).Default(); } + bool Uint(unsigned) { return static_cast(*this).Default(); } + bool Int64(int64_t) { return static_cast(*this).Default(); } + bool Uint64(uint64_t) { return static_cast(*this).Default(); } + bool Double(double) { return static_cast(*this).Default(); } + bool String(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } + bool StartObject() { return static_cast(*this).Default(); } + bool Key(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } + bool EndObject(SizeType) { return static_cast(*this).Default(); } + bool StartArray() { return static_cast(*this).Default(); } + bool EndArray(SizeType) { return static_cast(*this).Default(); } +}; + +/////////////////////////////////////////////////////////////////////////////// +// StreamLocalCopy + +namespace internal { + +template::copyOptimization> +class StreamLocalCopy; + +//! Do copy optimization. +template +class StreamLocalCopy { +public: + StreamLocalCopy(Stream& original) : s(original), original_(original) {} + ~StreamLocalCopy() { original_ = s; } + + Stream s; + +private: + StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; + + Stream& original_; +}; + +//! Keep reference. +template +class StreamLocalCopy { +public: + StreamLocalCopy(Stream& original) : s(original) {} + + Stream& s; + +private: + StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; +}; + +} // namespace internal + +/////////////////////////////////////////////////////////////////////////////// +// SkipWhitespace + +//! Skip the JSON white spaces in a stream. +/*! \param is A input stream for skipping white spaces. + \note This function has SSE2/SSE4.2 specialization. +*/ +template +void SkipWhitespace(InputStream& is) { + internal::StreamLocalCopy copy(is); + InputStream& s(copy.s); + + while (s.Peek() == ' ' || s.Peek() == '\n' || s.Peek() == '\r' || s.Peek() == '\t') + s.Take(); +} + +#ifdef RAPIDJSON_SSE42 +//! Skip whitespace with SSE 4.2 pcmpistrm instruction, testing 16 8-byte characters at once. +inline const char *SkipWhitespace_SIMD(const char* p) { + // Fast return for single non-whitespace + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // 16-byte align to the next boundary + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & ~15); + while (p != nextAligned) + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // The rest of string using SIMD + static const char whitespace[16] = " \n\r\t"; + const __m128i w = _mm_loadu_si128((const __m128i *)&whitespace[0]); + + for (;; p += 16) { + const __m128i s = _mm_load_si128((const __m128i *)p); + const unsigned r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); + if (r != 0) { // some of characters is non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } + } +} + +#elif defined(RAPIDJSON_SSE2) + +//! Skip whitespace with SSE2 instructions, testing 16 8-byte characters at once. +inline const char *SkipWhitespace_SIMD(const char* p) { + // Fast return for single non-whitespace + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // 16-byte align to the next boundary + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & ~15); + while (p != nextAligned) + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // The rest of string + static const char whitespaces[4][17] = { + " ", + "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r", + "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"}; + + const __m128i w0 = _mm_loadu_si128((const __m128i *)&whitespaces[0][0]); + const __m128i w1 = _mm_loadu_si128((const __m128i *)&whitespaces[1][0]); + const __m128i w2 = _mm_loadu_si128((const __m128i *)&whitespaces[2][0]); + const __m128i w3 = _mm_loadu_si128((const __m128i *)&whitespaces[3][0]); + + for (;; p += 16) { + const __m128i s = _mm_load_si128((const __m128i *)p); + __m128i x = _mm_cmpeq_epi8(s, w0); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); + unsigned short r = (unsigned short)~_mm_movemask_epi8(x); + if (r != 0) { // some of characters may be non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } + } +} + +#endif // RAPIDJSON_SSE2 + +#ifdef RAPIDJSON_SIMD +//! Template function specialization for InsituStringStream +template<> inline void SkipWhitespace(InsituStringStream& is) { + is.src_ = const_cast(SkipWhitespace_SIMD(is.src_)); +} + +//! Template function specialization for StringStream +template<> inline void SkipWhitespace(StringStream& is) { + is.src_ = SkipWhitespace_SIMD(is.src_); +} +#endif // RAPIDJSON_SIMD + +/////////////////////////////////////////////////////////////////////////////// +// GenericReader + +//! SAX-style JSON parser. Use \ref Reader for UTF8 encoding and default allocator. +/*! GenericReader parses JSON text from a stream, and send events synchronously to an + object implementing Handler concept. + + It needs to allocate a stack for storing a single decoded string during + non-destructive parsing. + + For in-situ parsing, the decoded string is directly written to the source + text string, no temporary buffer is required. + + A GenericReader object can be reused for parsing multiple JSON text. + + \tparam SourceEncoding Encoding of the input stream. + \tparam TargetEncoding Encoding of the parse output. + \tparam StackAllocator Allocator type for stack. +*/ +template +class GenericReader { +public: + typedef typename SourceEncoding::Ch Ch; //!< SourceEncoding character type + + //! Constructor. + /*! \param allocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing) + \param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing) + */ + GenericReader(StackAllocator* stackAllocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(stackAllocator, stackCapacity), parseResult_() {} + + //! Parse JSON text. + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept. + \tparam Handler Type of handler, implementing Handler concept. + \param is Input stream to be parsed. + \param handler The handler to receive events. + \return Whether the parsing is successful. + */ + template + ParseResult Parse(InputStream& is, Handler& handler) { + if (parseFlags & kParseIterativeFlag) + return IterativeParse(is, handler); + + parseResult_.Clear(); + + ClearStackOnExit scope(*this); + + SkipWhitespace(is); + + if (is.Peek() == '\0') { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell()); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + } + else { + ParseValue(is, handler); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + + if (!(parseFlags & kParseStopWhenDoneFlag)) { + SkipWhitespace(is); + + if (is.Peek() != '\0') { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + } + } + } + + return parseResult_; + } + + //! Parse JSON text (with \ref kParseDefaultFlags) + /*! \tparam InputStream Type of input stream, implementing Stream concept + \tparam Handler Type of handler, implementing Handler concept. + \param is Input stream to be parsed. + \param handler The handler to receive events. + \return Whether the parsing is successful. + */ + template + ParseResult Parse(InputStream& is, Handler& handler) { + return Parse(is, handler); + } + + //! Whether a parse error has occured in the last parsing. + bool HasParseError() const { return parseResult_.IsError(); } + + //! Get the \ref ParseErrorCode of last parsing. + ParseErrorCode GetParseErrorCode() const { return parseResult_.Code(); } + + //! Get the position of last parsing error in input, 0 otherwise. + size_t GetErrorOffset() const { return parseResult_.Offset(); } + +protected: + void SetParseError(ParseErrorCode code, size_t offset) { parseResult_.Set(code, offset); } + +private: + // Prohibit copy constructor & assignment operator. + GenericReader(const GenericReader&); + GenericReader& operator=(const GenericReader&); + + void ClearStack() { stack_.Clear(); } + + // clear stack on any exit from ParseStream, e.g. due to exception + struct ClearStackOnExit { + explicit ClearStackOnExit(GenericReader& r) : r_(r) {} + ~ClearStackOnExit() { r_.ClearStack(); } + private: + GenericReader& r_; + ClearStackOnExit(const ClearStackOnExit&); + ClearStackOnExit& operator=(const ClearStackOnExit&); + }; + + // Parse object: { string : value, ... } + template + void ParseObject(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == '{'); + is.Take(); // Skip '{' + + if (!handler.StartObject()) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + + SkipWhitespace(is); + + if (is.Peek() == '}') { + is.Take(); + if (!handler.EndObject(0)) // empty object + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } + + for (SizeType memberCount = 0;;) { + if (is.Peek() != '"') + RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); + + ParseString(is, handler, true); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + SkipWhitespace(is); + + if (is.Take() != ':') + RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); + + SkipWhitespace(is); + + ParseValue(is, handler); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + SkipWhitespace(is); + + ++memberCount; + + switch (is.Take()) { + case ',': SkipWhitespace(is); break; + case '}': + if (!handler.EndObject(memberCount)) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + else + return; + default: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); + } + } + } + + // Parse array: [ value, ... ] + template + void ParseArray(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == '['); + is.Take(); // Skip '[' + + if (!handler.StartArray()) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + + SkipWhitespace(is); + + if (is.Peek() == ']') { + is.Take(); + if (!handler.EndArray(0)) // empty array + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } + + for (SizeType elementCount = 0;;) { + ParseValue(is, handler); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + ++elementCount; + SkipWhitespace(is); + + switch (is.Take()) { + case ',': SkipWhitespace(is); break; + case ']': + if (!handler.EndArray(elementCount)) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + else + return; + default: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); + } + } + } + + template + void ParseNull(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == 'n'); + is.Take(); + + if (is.Take() == 'u' && is.Take() == 'l' && is.Take() == 'l') { + if (!handler.Null()) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); + } + + template + void ParseTrue(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == 't'); + is.Take(); + + if (is.Take() == 'r' && is.Take() == 'u' && is.Take() == 'e') { + if (!handler.Bool(true)) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); + } + + template + void ParseFalse(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == 'f'); + is.Take(); + + if (is.Take() == 'a' && is.Take() == 'l' && is.Take() == 's' && is.Take() == 'e') { + if (!handler.Bool(false)) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); + } + + // Helper function to parse four hexidecimal digits in \uXXXX in ParseString(). + template + unsigned ParseHex4(InputStream& is) { + unsigned codepoint = 0; + for (int i = 0; i < 4; i++) { + Ch c = is.Take(); + codepoint <<= 4; + codepoint += static_cast(c); + if (c >= '0' && c <= '9') + codepoint -= '0'; + else if (c >= 'A' && c <= 'F') + codepoint -= 'A' - 10; + else if (c >= 'a' && c <= 'f') + codepoint -= 'a' - 10; + else { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, is.Tell() - 1); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(0); + } + } + return codepoint; + } + + class StackStream { + public: + typedef typename TargetEncoding::Ch Ch; + + StackStream(internal::Stack& stack) : stack_(stack), length_(0) {} + RAPIDJSON_FORCEINLINE void Put(Ch c) { + *stack_.template Push() = c; + ++length_; + } + internal::Stack& stack_; + SizeType length_; + + private: + StackStream(const StackStream&); + StackStream& operator=(const StackStream&); + }; + + // Parse string and generate String event. Different code paths for kParseInsituFlag. + template + void ParseString(InputStream& is, Handler& handler, bool isKey = false) { + internal::StreamLocalCopy copy(is); + InputStream& s(copy.s); + + bool success = false; + if (parseFlags & kParseInsituFlag) { + typename InputStream::Ch *head = s.PutBegin(); + ParseStringToStream(s, s); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + size_t length = s.PutEnd(head) - 1; + RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); + const typename TargetEncoding::Ch* const str = (typename TargetEncoding::Ch*)head; + success = (isKey ? handler.Key(str, SizeType(length), false) : handler.String(str, SizeType(length), false)); + } + else { + StackStream stackStream(stack_); + ParseStringToStream(s, stackStream); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + const typename TargetEncoding::Ch* const str = stack_.template Pop(stackStream.length_); + success = (isKey ? handler.Key(str, stackStream.length_ - 1, true) : handler.String(str, stackStream.length_ - 1, true)); + } + if (!success) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); + } + + // Parse string to an output is + // This function handles the prefix/suffix double quotes, escaping, and optional encoding validation. + template + RAPIDJSON_FORCEINLINE void ParseStringToStream(InputStream& is, OutputStream& os) { +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + static const char escape[256] = { + Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', + Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, + 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, + 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 + }; +#undef Z16 +//!@endcond + + RAPIDJSON_ASSERT(is.Peek() == '\"'); + is.Take(); // Skip '\"' + + for (;;) { + Ch c = is.Peek(); + if (c == '\\') { // Escape + is.Take(); + Ch e = is.Take(); + if ((sizeof(Ch) == 1 || unsigned(e) < 256) && escape[(unsigned char)e]) { + os.Put(escape[(unsigned char)e]); + } + else if (e == 'u') { // Unicode + unsigned codepoint = ParseHex4(is); + if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { + // Handle UTF-16 surrogate pair + if (is.Take() != '\\' || is.Take() != 'u') + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, is.Tell() - 2); + unsigned codepoint2 = ParseHex4(is); + if (codepoint2 < 0xDC00 || codepoint2 > 0xDFFF) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, is.Tell() - 2); + codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; + } + TEncoding::Encode(os, codepoint); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell() - 1); + } + else if (c == '"') { // Closing double quote + is.Take(); + os.Put('\0'); // null-terminate the string + return; + } + else if (c == '\0') + RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell() - 1); + else if ((unsigned)c < 0x20) // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF + RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell() - 1); + else { + if (parseFlags & kParseValidateEncodingFlag ? + !Transcoder::Validate(is, os) : + !Transcoder::Transcode(is, os)) + RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, is.Tell()); + } + } + } + + inline double StrtodFastPath(double significand, int exp) { + // Fast path only works on limited range of values. + // But for simplicity and performance, currently only implement this. + // see http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + if (exp < -308) + return 0.0; + else if (exp >= 0) + return significand * internal::Pow10(exp); + else + return significand / internal::Pow10(-exp); + } + + template + void ParseNumber(InputStream& is, Handler& handler) { + internal::StreamLocalCopy copy(is); + InputStream& s(copy.s); + + // Parse minus + bool minus = false; + if (s.Peek() == '-') { + minus = true; + s.Take(); + } + + // Parse int: zero / ( digit1-9 *DIGIT ) + unsigned i = 0; + uint64_t i64 = 0; + bool use64bit = false; + if (s.Peek() == '0') { + i = 0; + s.Take(); + } + else if (s.Peek() >= '1' && s.Peek() <= '9') { + i = static_cast(s.Take() - '0'); + + if (minus) + while (s.Peek() >= '0' && s.Peek() <= '9') { + if (i >= 214748364) { // 2^31 = 2147483648 + if (i != 214748364 || s.Peek() > '8') { + i64 = i; + use64bit = true; + break; + } + } + i = i * 10 + static_cast(s.Take() - '0'); + } + else + while (s.Peek() >= '0' && s.Peek() <= '9') { + if (i >= 429496729) { // 2^32 - 1 = 4294967295 + if (i != 429496729 || s.Peek() > '5') { + i64 = i; + use64bit = true; + break; + } + } + i = i * 10 + static_cast(s.Take() - '0'); + } + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + + // Parse 64bit int + double d = 0.0; + bool useDouble = false; + if (use64bit) { + if (minus) + while (s.Peek() >= '0' && s.Peek() <= '9') { + if (i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC)) // 2^63 = 9223372036854775808 + if (i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8') { + d = (double)i64; + useDouble = true; + break; + } + i64 = i64 * 10 + static_cast(s.Take() - '0'); + } + else + while (s.Peek() >= '0' && s.Peek() <= '9') { + if (i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999)) // 2^64 - 1 = 18446744073709551615 + if (i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5') { + d = (double)i64; + useDouble = true; + break; + } + i64 = i64 * 10 + static_cast(s.Take() - '0'); + } + } + + // Force double for big integer + if (useDouble) { + while (s.Peek() >= '0' && s.Peek() <= '9') { + if (d >= 1.7976931348623157e307) // DBL_MAX / 10.0 + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, s.Tell()); + d = d * 10 + (s.Take() - '0'); + } + } + + // Parse frac = decimal-point 1*DIGIT + int expFrac = 0; + if (s.Peek() == '.') { + s.Take(); + +#if RAPIDJSON_64BIT + // Use i64 to store significand in 64-bit architecture + if (!useDouble) { + if (!use64bit) + i64 = i; + + while (s.Peek() >= '0' && s.Peek() <= '9') { + if (i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999)) + break; + else { + i64 = i64 * 10 + static_cast(s.Take() - '0'); + --expFrac; + } + } + + d = (double)i64; + } +#else + // Use double to store significand in 32-bit architecture + if (!useDouble) + d = use64bit ? (double)i64 : (double)i; +#endif + useDouble = true; + + while (s.Peek() >= '0' && s.Peek() <= '9') { + d = d * 10 + (s.Take() - '0'); + --expFrac; + } + + if (expFrac == 0) + RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissFraction, s.Tell()); + } + + // Parse exp = e [ minus / plus ] 1*DIGIT + int exp = 0; + if (s.Peek() == 'e' || s.Peek() == 'E') { + if (!useDouble) { + d = use64bit ? (double)i64 : (double)i; + useDouble = true; + } + s.Take(); + + bool expMinus = false; + if (s.Peek() == '+') + s.Take(); + else if (s.Peek() == '-') { + s.Take(); + expMinus = true; + } + + if (s.Peek() >= '0' && s.Peek() <= '9') { + exp = s.Take() - '0'; + while (s.Peek() >= '0' && s.Peek() <= '9') { + exp = exp * 10 + (s.Take() - '0'); + if (exp > 308 && !expMinus) // exp > 308 should be rare, so it should be checked first. + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, s.Tell()); + } + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissExponent, s.Tell()); + + if (expMinus) + exp = -exp; + } + + // Finish parsing, call event according to the type of number. + bool cont = true; + if (useDouble) { + int expSum = exp + expFrac; + if (expSum < -308) { + // Prevent expSum < -308, making Pow10(expSum) = 0 + d = StrtodFastPath(d, exp); + d = StrtodFastPath(d, expFrac); + } + else + d = StrtodFastPath(d, expSum); + + cont = handler.Double(minus ? -d : d); + } + else { + if (use64bit) { + if (minus) + cont = handler.Int64(-(int64_t)i64); + else + cont = handler.Uint64(i64); + } + else { + if (minus) + cont = handler.Int(-(int)i); + else + cont = handler.Uint(i); + } + } + if (!cont) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); + } + + // Parse any JSON value + template + void ParseValue(InputStream& is, Handler& handler) { + switch (is.Peek()) { + case 'n': ParseNull (is, handler); break; + case 't': ParseTrue (is, handler); break; + case 'f': ParseFalse (is, handler); break; + case '"': ParseString(is, handler); break; + case '{': ParseObject(is, handler); break; + case '[': ParseArray (is, handler); break; + default : ParseNumber(is, handler); + } + } + + // Iterative Parsing + + // States + enum IterativeParsingState { + IterativeParsingStartState = 0, + IterativeParsingFinishState, + IterativeParsingErrorState, + + // Object states + IterativeParsingObjectInitialState, + IterativeParsingMemberKeyState, + IterativeParsingKeyValueDelimiterState, + IterativeParsingMemberValueState, + IterativeParsingMemberDelimiterState, + IterativeParsingObjectFinishState, + + // Array states + IterativeParsingArrayInitialState, + IterativeParsingElementState, + IterativeParsingElementDelimiterState, + IterativeParsingArrayFinishState, + + // Single value state + IterativeParsingValueState, + + cIterativeParsingStateCount + }; + + // Tokens + enum Token { + LeftBracketToken = 0, + RightBracketToken, + + LeftCurlyBracketToken, + RightCurlyBracketToken, + + CommaToken, + ColonToken, + + StringToken, + FalseToken, + TrueToken, + NullToken, + NumberToken, + + kTokenCount + }; + + RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) { + +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#define N NumberToken +#define N16 N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N + // Maps from ASCII to Token + static const unsigned char tokenMap[256] = { + N16, // 00~0F + N16, // 10~1F + N, N, StringToken, N, N, N, N, N, N, N, N, N, CommaToken, N, N, N, // 20~2F + N, N, N, N, N, N, N, N, N, N, ColonToken, N, N, N, N, N, // 30~3F + N16, // 40~4F + N, N, N, N, N, N, N, N, N, N, N, LeftBracketToken, N, RightBracketToken, N, N, // 50~5F + N, N, N, N, N, N, FalseToken, N, N, N, N, N, N, N, NullToken, N, // 60~6F + N, N, N, N, TrueToken, N, N, N, N, N, N, LeftCurlyBracketToken, N, RightCurlyBracketToken, N, N, // 70~7F + N16, N16, N16, N16, N16, N16, N16, N16 // 80~FF + }; +#undef N +#undef N16 +//!@endcond + + if (sizeof(Ch) == 1 || unsigned(c) < 256) + return (Token)tokenMap[(unsigned char)c]; + else + return NumberToken; + } + + RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) { + // current state x one lookahead token -> new state + static const char G[cIterativeParsingStateCount][kTokenCount] = { + // Start + { + IterativeParsingArrayInitialState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingValueState, // String + IterativeParsingValueState, // False + IterativeParsingValueState, // True + IterativeParsingValueState, // Null + IterativeParsingValueState // Number + }, + // Finish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Error(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // ObjectInitial + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberKeyState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // MemberKey + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingKeyValueDelimiterState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // KeyValueDelimiter + { + IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberValueState, // String + IterativeParsingMemberValueState, // False + IterativeParsingMemberValueState, // True + IterativeParsingMemberValueState, // Null + IterativeParsingMemberValueState // Number + }, + // MemberValue + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingMemberDelimiterState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // MemberDelimiter + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberKeyState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // ObjectFinish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // ArrayInitial + { + IterativeParsingArrayInitialState, // Left bracket(push Element state) + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push Element state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingElementState, // String + IterativeParsingElementState, // False + IterativeParsingElementState, // True + IterativeParsingElementState, // Null + IterativeParsingElementState // Number + }, + // Element + { + IterativeParsingErrorState, // Left bracket + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingElementDelimiterState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // ElementDelimiter + { + IterativeParsingArrayInitialState, // Left bracket(push Element state) + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push Element state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingElementState, // String + IterativeParsingElementState, // False + IterativeParsingElementState, // True + IterativeParsingElementState, // Null + IterativeParsingElementState // Number + }, + // ArrayFinish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Single Value (sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + } + }; // End of G + + return (IterativeParsingState)G[state][token]; + } + + // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). + // May return a new state on state pop. + template + RAPIDJSON_FORCEINLINE IterativeParsingState Transit(IterativeParsingState src, Token token, IterativeParsingState dst, InputStream& is, Handler& handler) { + switch (dst) { + case IterativeParsingStartState: + RAPIDJSON_ASSERT(false); + return IterativeParsingErrorState; + + case IterativeParsingFinishState: + return dst; + + case IterativeParsingErrorState: + return dst; + + case IterativeParsingObjectInitialState: + case IterativeParsingArrayInitialState: + { + // Push the state(Element or MemeberValue) if we are nested in another array or value of member. + // In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop. + IterativeParsingState n = src; + if (src == IterativeParsingArrayInitialState || src == IterativeParsingElementDelimiterState) + n = IterativeParsingElementState; + else if (src == IterativeParsingKeyValueDelimiterState) + n = IterativeParsingMemberValueState; + // Push current state. + *stack_.template Push(1) = n; + // Initialize and push the member/element count. + *stack_.template Push(1) = 0; + // Call handler + bool hr = (dst == IterativeParsingObjectInitialState) ? handler.StartObject() : handler.StartArray(); + // On handler short circuits the parsing. + if (!hr) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; + } + else { + is.Take(); + return dst; + } + } + + case IterativeParsingMemberKeyState: + ParseString(is, handler, true); + if (HasParseError()) + return IterativeParsingErrorState; + else + return dst; + + case IterativeParsingKeyValueDelimiterState: + if (token == ColonToken) { + is.Take(); + return dst; + } + else + return IterativeParsingErrorState; + + case IterativeParsingMemberValueState: + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) { + return IterativeParsingErrorState; + } + return dst; + + case IterativeParsingElementState: + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) { + return IterativeParsingErrorState; + } + return dst; + + case IterativeParsingMemberDelimiterState: + case IterativeParsingElementDelimiterState: + is.Take(); + // Update member/element count. + *stack_.template Top() = *stack_.template Top() + 1; + return dst; + + case IterativeParsingObjectFinishState: + { + // Get member count. + SizeType c = *stack_.template Pop(1); + // If the object is not empty, count the last member. + if (src == IterativeParsingMemberValueState) + ++c; + // Restore the state. + IterativeParsingState n = static_cast(*stack_.template Pop(1)); + // Transit to Finish state if this is the topmost scope. + if (n == IterativeParsingStartState) + n = IterativeParsingFinishState; + // Call handler + bool hr = handler.EndObject(c); + // On handler short circuits the parsing. + if (!hr) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; + } + else { + is.Take(); + return n; + } + } + + case IterativeParsingArrayFinishState: + { + // Get element count. + SizeType c = *stack_.template Pop(1); + // If the array is not empty, count the last element. + if (src == IterativeParsingElementState) + ++c; + // Restore the state. + IterativeParsingState n = static_cast(*stack_.template Pop(1)); + // Transit to Finish state if this is the topmost scope. + if (n == IterativeParsingStartState) + n = IterativeParsingFinishState; + // Call handler + bool hr = handler.EndArray(c); + // On handler short circuits the parsing. + if (!hr) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; + } + else { + is.Take(); + return n; + } + } + + case IterativeParsingValueState: + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) { + return IterativeParsingErrorState; + } + return IterativeParsingFinishState; + + default: + RAPIDJSON_ASSERT(false); + return IterativeParsingErrorState; + } + } + + template + void HandleError(IterativeParsingState src, InputStream& is) { + if (HasParseError()) { + // Error flag has been set. + return; + } + + switch (src) { + case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); + case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); + case IterativeParsingObjectInitialState: + case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); + case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); + case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); + case IterativeParsingElementState: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); + default: RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + } + } + + template + ParseResult IterativeParse(InputStream& is, Handler& handler) { + parseResult_.Clear(); + ClearStackOnExit scope(*this); + IterativeParsingState state = IterativeParsingStartState; + + SkipWhitespace(is); + while (is.Peek() != '\0') { + Token t = Tokenize(is.Peek()); + IterativeParsingState n = Predict(state, t); + IterativeParsingState d = Transit(state, t, n, is, handler); + + if (d == IterativeParsingErrorState) { + HandleError(state, is); + break; + } + + state = d; + + // Do not further consume streams if a root JSON has been parsed. + if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) + break; + + SkipWhitespace(is); + } + + // Handle the end of file. + if (state != IterativeParsingFinishState) + HandleError(state, is); + + return parseResult_; + } + + static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. + internal::Stack stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. + ParseResult parseResult_; +}; // class GenericReader + +//! Reader with UTF8 encoding and default allocator. +typedef GenericReader, UTF8<> > Reader; + +} // namespace rapidjson + +#ifdef _MSC_VER +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_READER_H_ diff --git a/tools/optimizer/rapidjson/stringbuffer.h b/tools/optimizer/rapidjson/stringbuffer.h new file mode 100644 index 0000000000000..55124f117adb9 --- /dev/null +++ b/tools/optimizer/rapidjson/stringbuffer.h @@ -0,0 +1,79 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_STRINGBUFFER_H_ +#define RAPIDJSON_STRINGBUFFER_H_ + +#include "rapidjson.h" +#include "internal/stack.h" + +namespace rapidjson { + +//! Represents an in-memory output stream. +/*! + \tparam Encoding Encoding of the stream. + \tparam Allocator type for allocating memory buffer. + \note implements Stream concept +*/ +template +struct GenericStringBuffer { + typedef typename Encoding::Ch Ch; + + GenericStringBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} + + void Put(Ch c) { *stack_.template Push() = c; } + void Flush() {} + + void Clear() { stack_.Clear(); } + void ShrinkToFit() { + // Push and pop a null terminator. This is safe. + *stack_.template Push() = '\0'; + stack_.ShrinkToFit(); + stack_.template Pop(1); + } + Ch* Push(size_t count) { return stack_.template Push(count); } + void Pop(size_t count) { stack_.template Pop(count); } + + const Ch* GetString() const { + // Push and pop a null terminator. This is safe. + *stack_.template Push() = '\0'; + stack_.template Pop(1); + + return stack_.template Bottom(); + } + + size_t GetSize() const { return stack_.GetSize(); } + + static const size_t kDefaultCapacity = 256; + mutable internal::Stack stack_; +}; + +//! String buffer with UTF8 encoding +typedef GenericStringBuffer > StringBuffer; + +//! Implement specialized version of PutN() with memset() for better performance. +template<> +inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { + std::memset(stream.stack_.Push(n), c, n * sizeof(c)); +} + +} // namespace rapidjson + +#endif // RAPIDJSON_STRINGBUFFER_H_ diff --git a/tools/optimizer/rapidjson/writer.h b/tools/optimizer/rapidjson/writer.h new file mode 100644 index 0000000000000..fb6601ef97684 --- /dev/null +++ b/tools/optimizer/rapidjson/writer.h @@ -0,0 +1,389 @@ +// Copyright (C) 2011 Milo Yip +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef RAPIDJSON_WRITER_H_ +#define RAPIDJSON_WRITER_H_ + +#include "rapidjson.h" +#include "internal/stack.h" +#include "internal/strfunc.h" +#include "internal/dtoa.h" +#include "internal/itoa.h" +#include "stringbuffer.h" +#include // placement new + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +#endif + +namespace rapidjson { + +//! JSON writer +/*! Writer implements the concept Handler. + It generates JSON text by events to an output os. + + User may programmatically calls the functions of a writer to generate JSON text. + + On the other side, a writer can also be passed to objects that generates events, + + for example Reader::Parse() and Document::Accept(). + + \tparam OutputStream Type of output stream. + \tparam SourceEncoding Encoding of source string. + \tparam TargetEncoding Encoding of output stream. + \tparam StackAllocator Type of allocator for allocating memory of stack. + \note implements Handler concept +*/ +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator> +class Writer { +public: + typedef typename SourceEncoding::Ch Ch; + + //! Constructor + /*! \param os Output stream. + \param allocator User supplied allocator. If it is null, it will create a private one. + \param levelDepth Initial capacity of stack. + */ + Writer(OutputStream& os, StackAllocator* stackAllocator = 0, size_t levelDepth = kDefaultLevelDepth) : + os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), hasRoot_(false) {} + + Writer(StackAllocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : + os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), hasRoot_(false) {} + + //! Reset the writer with a new stream. + /*! + This function reset the writer with a new stream and default settings, + in order to make a Writer object reusable for output multiple JSONs. + + \param os New output stream. + \code + Writer writer(os1); + writer.StartObject(); + // ... + writer.EndObject(); + + writer.Reset(os2); + writer.StartObject(); + // ... + writer.EndObject(); + \endcode + */ + void Reset(OutputStream& os) { + os_ = &os; + hasRoot_ = false; + level_stack_.Clear(); + } + + //! Checks whether the output is a complete JSON. + /*! + A complete JSON has a complete root object or array. + */ + bool IsComplete() const { + return hasRoot_ && level_stack_.Empty(); + } + + /*!@name Implementation of Handler + \see Handler + */ + //@{ + + bool Null() { Prefix(kNullType); return WriteNull(); } + bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return WriteBool(b); } + bool Int(int i) { Prefix(kNumberType); return WriteInt(i); } + bool Uint(unsigned u) { Prefix(kNumberType); return WriteUint(u); } + bool Int64(int64_t i64) { Prefix(kNumberType); return WriteInt64(i64); } + bool Uint64(uint64_t u64) { Prefix(kNumberType); return WriteUint64(u64); } + + //! Writes the given \c double value to the stream + /*! + \param d The value to be written. + \return Whether it is succeed. + */ + bool Double(double d) { Prefix(kNumberType); return WriteDouble(d); } + + bool String(const Ch* str, SizeType length, bool copy = false) { + (void)copy; + Prefix(kStringType); + return WriteString(str, length); + } + + bool StartObject() { + Prefix(kObjectType); + new (level_stack_.template Push()) Level(false); + return WriteStartObject(); + } + + bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + + bool EndObject(SizeType memberCount = 0) { + (void)memberCount; + RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); + RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); + level_stack_.template Pop(1); + bool ret = WriteEndObject(); + if (level_stack_.Empty()) // end of json text + os_->Flush(); + return ret; + } + + bool StartArray() { + Prefix(kArrayType); + new (level_stack_.template Push()) Level(true); + return WriteStartArray(); + } + + bool EndArray(SizeType elementCount = 0) { + (void)elementCount; + RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); + RAPIDJSON_ASSERT(level_stack_.template Top()->inArray); + level_stack_.template Pop(1); + bool ret = WriteEndArray(); + if (level_stack_.Empty()) // end of json text + os_->Flush(); + return ret; + } + //@} + + /*! @name Convenience extensions */ + //@{ + + //! Simpler but slower overload. + bool String(const Ch* str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } + + //@} + +protected: + //! Information for each nested level + struct Level { + Level(bool inArray_) : valueCount(0), inArray(inArray_) {} + size_t valueCount; //!< number of values in this level + bool inArray; //!< true if in array, otherwise in object + }; + + static const size_t kDefaultLevelDepth = 32; + + bool WriteNull() { + os_->Put('n'); os_->Put('u'); os_->Put('l'); os_->Put('l'); return true; + } + + bool WriteBool(bool b) { + if (b) { + os_->Put('t'); os_->Put('r'); os_->Put('u'); os_->Put('e'); + } + else { + os_->Put('f'); os_->Put('a'); os_->Put('l'); os_->Put('s'); os_->Put('e'); + } + return true; + } + + bool WriteInt(int i) { + char buffer[11]; + const char* end = internal::i32toa(i, buffer); + for (const char* p = buffer; p != end; ++p) + os_->Put(*p); + return true; + } + + bool WriteUint(unsigned u) { + char buffer[10]; + const char* end = internal::u32toa(u, buffer); + for (const char* p = buffer; p != end; ++p) + os_->Put(*p); + return true; + } + + bool WriteInt64(int64_t i64) { + char buffer[21]; + const char* end = internal::i64toa(i64, buffer); + for (const char* p = buffer; p != end; ++p) + os_->Put(*p); + return true; + } + + bool WriteUint64(uint64_t u64) { + char buffer[20]; + char* end = internal::u64toa(u64, buffer); + for (char* p = buffer; p != end; ++p) + os_->Put(*p); + return true; + } + + bool WriteDouble(double d) { + char buffer[25]; + char* end = internal::dtoa(d, buffer); + for (char* p = buffer; p != end; ++p) + os_->Put(*p); + return true; + } + + bool WriteString(const Ch* str, SizeType length) { + static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + static const char escape[256] = { +#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + //0 1 2 3 4 5 6 7 8 9 A B C D E F + 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'b', 't', 'n', 'u', 'f', 'r', 'u', 'u', // 00 + 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', // 10 + 0, 0, '"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20 + Z16, Z16, // 30~4F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, // 50 + Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 // 60~FF +#undef Z16 + }; + + os_->Put('\"'); + GenericStringStream is(str); + while (is.Tell() < length) { + const Ch c = is.Peek(); + if (!TargetEncoding::supportUnicode && (unsigned)c >= 0x80) { + // Unicode escaping + unsigned codepoint; + if (!SourceEncoding::Decode(is, &codepoint)) + return false; + os_->Put('\\'); + os_->Put('u'); + if (codepoint <= 0xD7FF || (codepoint >= 0xE000 && codepoint <= 0xFFFF)) { + os_->Put(hexDigits[(codepoint >> 12) & 15]); + os_->Put(hexDigits[(codepoint >> 8) & 15]); + os_->Put(hexDigits[(codepoint >> 4) & 15]); + os_->Put(hexDigits[(codepoint ) & 15]); + } + else if (codepoint >= 0x010000 && codepoint <= 0x10FFFF) { + // Surrogate pair + unsigned s = codepoint - 0x010000; + unsigned lead = (s >> 10) + 0xD800; + unsigned trail = (s & 0x3FF) + 0xDC00; + os_->Put(hexDigits[(lead >> 12) & 15]); + os_->Put(hexDigits[(lead >> 8) & 15]); + os_->Put(hexDigits[(lead >> 4) & 15]); + os_->Put(hexDigits[(lead ) & 15]); + os_->Put('\\'); + os_->Put('u'); + os_->Put(hexDigits[(trail >> 12) & 15]); + os_->Put(hexDigits[(trail >> 8) & 15]); + os_->Put(hexDigits[(trail >> 4) & 15]); + os_->Put(hexDigits[(trail ) & 15]); + } + else + return false; // invalid code point + } + else if ((sizeof(Ch) == 1 || (unsigned)c < 256) && escape[(unsigned char)c]) { + is.Take(); + os_->Put('\\'); + os_->Put(escape[(unsigned char)c]); + if (escape[(unsigned char)c] == 'u') { + os_->Put('0'); + os_->Put('0'); + os_->Put(hexDigits[(unsigned char)c >> 4]); + os_->Put(hexDigits[(unsigned char)c & 0xF]); + } + } + else + Transcoder::Transcode(is, *os_); + } + os_->Put('\"'); + return true; + } + + bool WriteStartObject() { os_->Put('{'); return true; } + bool WriteEndObject() { os_->Put('}'); return true; } + bool WriteStartArray() { os_->Put('['); return true; } + bool WriteEndArray() { os_->Put(']'); return true; } + + void Prefix(Type type) { + (void)type; + if (level_stack_.GetSize() != 0) { // this value is not at root + Level* level = level_stack_.template Top(); + if (level->valueCount > 0) { + if (level->inArray) + os_->Put(','); // add comma if it is not the first element in array + else // in object + os_->Put((level->valueCount % 2 == 0) ? ',' : ':'); + } + if (!level->inArray && level->valueCount % 2 == 0) + RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name + level->valueCount++; + } + else { + RAPIDJSON_ASSERT(!hasRoot_); // Should only has one and only one root. + hasRoot_ = true; + } + } + + OutputStream* os_; + internal::Stack level_stack_; + bool hasRoot_; + +private: + // Prohibit copy constructor & assignment operator. + Writer(const Writer&); + Writer& operator=(const Writer&); +}; + +// Full specialization for StringStream to prevent memory copying + +template<> +inline bool Writer::WriteInt(int i) { + char *buffer = os_->Push(11); + const char* end = internal::i32toa(i, buffer); + os_->Pop(11 - (end - buffer)); + return true; +} + +template<> +inline bool Writer::WriteUint(unsigned u) { + char *buffer = os_->Push(10); + const char* end = internal::u32toa(u, buffer); + os_->Pop(10 - (end - buffer)); + return true; +} + +template<> +inline bool Writer::WriteInt64(int64_t i64) { + char *buffer = os_->Push(21); + const char* end = internal::i64toa(i64, buffer); + os_->Pop(21 - (end - buffer)); + return true; +} + +template<> +inline bool Writer::WriteUint64(uint64_t u) { + char *buffer = os_->Push(20); + const char* end = internal::u64toa(u, buffer); + os_->Pop(20 - (end - buffer)); + return true; +} + +template<> +inline bool Writer::WriteDouble(double d) { + char *buffer = os_->Push(25); + char* end = internal::dtoa(d, buffer); + os_->Pop(25 - (end - buffer)); + return true; +} + +} // namespace rapidjson + +#ifdef _MSC_VER +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_RAPIDJSON_H_ From 38f11ca42c3dacaec5185f754fc61da5653076d2 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 30 Oct 2014 10:30:58 -0700 Subject: [PATCH 004/161] compiling early version of optimizer.cpp --- tools/optimizer/optimizer.cpp | 114 ++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 tools/optimizer/optimizer.cpp diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp new file mode 100644 index 0000000000000..40ce21a3f9ff4 --- /dev/null +++ b/tools/optimizer/optimizer.cpp @@ -0,0 +1,114 @@ +#include +#include +#include +#include + +#include +#include + +#include "rapidjson/document.h" +#include "rapidjson/writer.h" +#include "rapidjson/stringbuffer.h" + +using namespace rapidjson; + +//================== +// Globals +//================== + +Document doc; + +//================== +// Infrastructure +//================== + +// Traverses the children of a node. +// visit() receives a reference, so it can modify the value being visited directly. +// TODO: stoppable version of this +void traverseChildren(Value &node, std::function visit) { + assert(node.IsArray()); + int size = node.Size(); + for (int i = 0; i < size; i++) { + Value &subnode = node[i]; + if (subnode.IsArray()) { + visit(subnode); + } + } +} + +Value makeName(const char *str) { + Value ret; + ret.SetArray(); + ret.PushBack(Value().SetString("name", 4), doc.GetAllocator()); + ret.PushBack(Value().SetString(str, strlen(str)), doc.GetAllocator()); + return ret; +} + +//================== +// Params +//================== + +bool asm_ = false; + +//===================== +// Optimization passes +//===================== + +void optimizeFrounds(Value &ast) { + // collapse fround(fround(..)), which can happen due to elimination + // also emit f0 instead of fround(0) (except in returns) + bool inReturn = false; + std::function fix = [&](Value& node) { + bool ret = node[0] == "return"; + if (ret) inReturn = true; + traverseChildren(node, fix); + if (ret) inReturn = false; + if (node[0] == "call" && node[1][0] == "name" && node[1][1] == "Math_fround") { + Value& arg = node[2][0]; + if (arg[0] == "num") { + if (!inReturn && arg[1] == 0) node = makeName("f0"); + } else if (arg[0] == "call" && arg[1][0] == "name" && arg[1][1] == "Math_fround") { + node = arg; + } + } + }; + traverseChildren(ast, fix); +} + +//================== +// Main +//================== + +int main(int argc, char **argv) { + // Read input file + char *input = argv[1]; + struct stat st; + int result = stat(input, &st); + assert(result == 0); + int size = st.st_size; + char *json = new char[size+1]; + FILE *f = fopen(input, "rb"); + fread(json, 1, size, f); + fclose(f); + json[size] = 0; + + // Parse JSON source into a Document + doc.Parse(json); + delete[] json; + + // Run passes on the Document + for (int i = 2; i < argc; i++) { + std::string str(argv[i]); + if (str == "asm") asm_ = true; + else if (str == "optimizeFrounds") optimizeFrounds(doc); + } + + // Emit JSON of modified Document + StringBuffer buffer; + Writer writer(buffer); + doc.Accept(writer); + puts(buffer.GetString()); + + return 0; +} + From 968d27db26fb62710fe0730d63ee45faccfe2253 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 30 Oct 2014 14:02:30 -0700 Subject: [PATCH 005/161] use global singletons for common strings --- tools/optimizer/optimizer.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 40ce21a3f9ff4..131ef69051ec0 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -18,6 +18,17 @@ using namespace rapidjson; Document doc; +class GlobalStringValue : public Value { +public: + GlobalStringValue(const char *str) : Value(str, strlen(str)) {} +}; + +GlobalStringValue RETURN("return"), + CALL("call"), + NAME("name"), + NUM("num"), + MATH_FROUND("Math_fround"); + //================== // Infrastructure //================== @@ -59,15 +70,15 @@ void optimizeFrounds(Value &ast) { // also emit f0 instead of fround(0) (except in returns) bool inReturn = false; std::function fix = [&](Value& node) { - bool ret = node[0] == "return"; + bool ret = node[0] == RETURN; if (ret) inReturn = true; traverseChildren(node, fix); if (ret) inReturn = false; - if (node[0] == "call" && node[1][0] == "name" && node[1][1] == "Math_fround") { + if (node[0] == CALL && node[1][0] == NAME && node[1][1] == MATH_FROUND) { Value& arg = node[2][0]; - if (arg[0] == "num") { + if (arg[0] == NUM) { if (!inReturn && arg[1] == 0) node = makeName("f0"); - } else if (arg[0] == "call" && arg[1][0] == "name" && arg[1][1] == "Math_fround") { + } else if (arg[0] == CALL && arg[1][0] == NAME && arg[1][1] == MATH_FROUND) { node = arg; } } From f96028b4068f77322cb14c83f2f17bcf11e19544 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 30 Oct 2014 14:18:18 -0700 Subject: [PATCH 006/161] refactor EMCC_DEBUG=2 code to reuse the normal mode code --- emcc | 50 ++++++++++++++------------------------------------ 1 file changed, 14 insertions(+), 36 deletions(-) diff --git a/emcc b/emcc index 3f6642714ab7a..6c2998814b1a8 100755 --- a/emcc +++ b/emcc @@ -1369,52 +1369,30 @@ try: js_optimizer_extra_info = {} js_optimizer_queue_history = [] js_optimizer_blacklist = (os.environ.get('EMCC_JSOPT_BLACKLIST') or '').split(',') - def flush_js_optimizer_queue(): + def flush_js_optimizer_queue(title='js_opts'): global final, js_optimizer_queue, js_optimizer_extra_info, js_optimizer_queue_history js_optimizer_queue = filter(lambda p: p not in js_optimizer_blacklist, js_optimizer_queue) - if len(js_optimizer_extra_info) == 0: + if js_optimizer_extra_info is not None and len(js_optimizer_extra_info) == 0: js_optimizer_extra_info = None - if len(js_optimizer_queue) > 0 and not(not shared.Settings.ASM_JS and len(js_optimizer_queue) == 1 and js_optimizer_queue[0] == 'last'): - def add_opt_args(args): + if len(js_optimizer_queue) > 0 and not(not shared.Settings.ASM_JS and len(js_optimizer_queue) == 1 and js_optimizer_queue[0] == 'last'): + passes = js_optimizer_queue[:] + if DEBUG != '2' or len(passes) < 2: if shared.Settings.ASM_JS: - args = ['asm'] + args + passes = ['asm'] + passes if shared.Settings.PRECISE_F32: - args = ['asmPreciseF32'] + args - return args - - if DEBUG != '2': - js_optimizer_queue = add_opt_args(js_optimizer_queue) - logging.debug('applying js optimization passes: %s', js_optimizer_queue) - final = shared.Building.js_optimizer(final, js_optimizer_queue, jcache, debug_level >= 4, js_optimizer_extra_info) + passes = ['asmPreciseF32'] + passes + passes = filter(lambda p: p != 'pretty', passes) + logging.debug('applying js optimization passes: %s', passes) + final = shared.Building.js_optimizer(final, passes, jcache, debug_level >= 4, js_optimizer_extra_info) js_transform_tempfiles.append(final) - if DEBUG: save_intermediate('js_opts') + if DEBUG: save_intermediate(title) else: - for i in range(len(js_optimizer_queue)): - name = js_optimizer_queue[i] - passes = [name] if name is not 'pretty' else [] - passes = add_opt_args(passes) - if shared.Settings.ASM_JS: - passes = ['asm'] + passes - just_split = False - just_concat = False - - # XXX experimental code to emit JSON in intermediate phases - if i > 0: - passes = ['receiveJSON'] + passes - just_split = True - if i < len(js_optimizer_queue)-1: - passes += ['emitJSON'] - just_concat = True - - logging.debug('applying js optimization pass: %s', passes) - t = time.time() - final = shared.Building.js_optimizer(final, passes, jcache, debug_level >= 4, js_optimizer_extra_info, just_split=just_split, just_concat=just_concat) - print >> sys.stderr, 'took', time.time()-t - js_transform_tempfiles.append(final) - save_intermediate(name) + for p in passes: + js_optimizer_queue = [p] + flush_js_optimizer_queue(p) js_optimizer_queue_history += js_optimizer_queue js_optimizer_queue = [] js_optimizer_extra_info = {} From 8ba0aecb2a30e32206541e1dfa61f5cbb44cbce4 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 30 Oct 2014 14:46:15 -0700 Subject: [PATCH 007/161] write code to bridge js and native optimizers, serializing to json between then --- emcc | 38 ++++++++++++++++++++++++++++++++++---- tools/js-optimizer.js | 1 + 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/emcc b/emcc index 6c2998814b1a8..60d8db0dee172 100755 --- a/emcc +++ b/emcc @@ -1369,6 +1369,8 @@ try: js_optimizer_extra_info = {} js_optimizer_queue_history = [] js_optimizer_blacklist = (os.environ.get('EMCC_JSOPT_BLACKLIST') or '').split(',') + NATIVE_OPTIMIZER_PASSES = set() # ['optimizeFrounds']) + def flush_js_optimizer_queue(title='js_opts'): global final, js_optimizer_queue, js_optimizer_extra_info, js_optimizer_queue_history @@ -1378,17 +1380,45 @@ try: js_optimizer_extra_info = None if len(js_optimizer_queue) > 0 and not(not shared.Settings.ASM_JS and len(js_optimizer_queue) == 1 and js_optimizer_queue[0] == 'last'): - passes = js_optimizer_queue[:] - if DEBUG != '2' or len(passes) < 2: + + def run_passes(passes): + global final if shared.Settings.ASM_JS: passes = ['asm'] + passes if shared.Settings.PRECISE_F32: passes = ['asmPreciseF32'] + passes - passes = filter(lambda p: p != 'pretty', passes) logging.debug('applying js optimization passes: %s', passes) final = shared.Building.js_optimizer(final, passes, jcache, debug_level >= 4, js_optimizer_extra_info) js_transform_tempfiles.append(final) if DEBUG: save_intermediate(title) + + passes = js_optimizer_queue[:] + + if DEBUG != '2' or len(passes) < 2: + # by assumption, our input is JS, and our output is JS. If a pass is going to run in the native optimizer in C++, then we + # must give it JSON and receive from it JSON + chunks = [] + curr = [] + native = False + for p in passes: + if p not in NATIVE_OPTIMIZER_PASSES: + if native: + chunks.append(curr) + curr = [] + native = False + curr.append('receiveJSON') + curr.append(p) + else: # p is native + if not native: + curr.append('emitJSON') + chunks.append(curr) + curr = [] + native = True + curr.append(p) + if len(curr) > 0: + chunks.append(curr) + for chunk in chunks: + run_passes(chunk) else: for p in passes: js_optimizer_queue = [p] @@ -1402,7 +1432,7 @@ try: if DEBUG == '2': # Clean up the syntax a bit - js_optimizer_queue += ['pretty'] + js_optimizer_queue += ['noop'] def get_eliminate(): if shared.Settings.ALLOW_MEMORY_GROWTH: diff --git a/tools/js-optimizer.js b/tools/js-optimizer.js index f348811370046..53a5ec583b709 100644 --- a/tools/js-optimizer.js +++ b/tools/js-optimizer.js @@ -7642,6 +7642,7 @@ var passes = { emterpretify: emterpretify, findReachable: findReachable, asmLastOpts: asmLastOpts, + noop: function() {}, // flags minifyWhitespace: function() { minifyWhitespace = true }, From 94d1346f5d5a1e6f088a1a0950eb62152471c11a Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 30 Oct 2014 15:13:57 -0700 Subject: [PATCH 008/161] handle just_split/just_concat when bridging js and native optimizers --- emcc | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/emcc b/emcc index 60d8db0dee172..d1e52f6f8a371 100755 --- a/emcc +++ b/emcc @@ -1381,14 +1381,14 @@ try: if len(js_optimizer_queue) > 0 and not(not shared.Settings.ASM_JS and len(js_optimizer_queue) == 1 and js_optimizer_queue[0] == 'last'): - def run_passes(passes): + def run_passes(passes, title, just_split, just_concat): global final if shared.Settings.ASM_JS: passes = ['asm'] + passes if shared.Settings.PRECISE_F32: passes = ['asmPreciseF32'] + passes logging.debug('applying js optimization passes: %s', passes) - final = shared.Building.js_optimizer(final, passes, jcache, debug_level >= 4, js_optimizer_extra_info) + final = shared.Building.js_optimizer(final, passes, jcache, debug_level >= 4, js_optimizer_extra_info, just_split=just_split, just_concat=just_concat) js_transform_tempfiles.append(final) if DEBUG: save_intermediate(title) @@ -1403,7 +1403,7 @@ try: for p in passes: if p not in NATIVE_OPTIMIZER_PASSES: if native: - chunks.append(curr) + chunks.append(['receiveJSON'] + curr + ['emitJSON']) curr = [] native = False curr.append('receiveJSON') @@ -1416,9 +1416,13 @@ try: native = True curr.append(p) if len(curr) > 0: + assert not native # FIXME, need to add JSON notes chunks.append(curr) - for chunk in chunks: - run_passes(chunk) + if len(chunks) == 1: + run_passes(chunks[0], title, just_split=False, just_concat=False) + else: + for i in range(len(chunks)): + run_passes(chunks[i], 'js_opts_' + str(i), just_split='receiveJSON' in chunks[i], just_concat='emitJSON' in chunks[i]) else: for p in passes: js_optimizer_queue = [p] From 1ff2cd29e160fbd4ae78b4b395475aaa63ba3161 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 30 Oct 2014 16:13:54 -0700 Subject: [PATCH 009/161] avoid copy in Cache.get if already in the right place --- tools/cache.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/cache.py b/tools/cache.py index 6f2443e8a958f..88b3859dce7d5 100644 --- a/tools/cache.py +++ b/tools/cache.py @@ -33,7 +33,9 @@ def get(self, shortname, creator, extension='.bc'): if os.path.exists(cachename): return cachename self.ensure() - shutil.copyfile(creator(), cachename) + temp = creator() + if temp != cachename: + shutil.copyfile(temp, cachename) return cachename # JS-specific cache. We cache the results of compilation and optimization, From f841398e9cb87321628fad6ee3d43bf6fb4b603d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 30 Oct 2014 16:14:17 -0700 Subject: [PATCH 010/161] infrastructure to build and use native optimizer --- emcc | 3 +-- tools/js_optimizer.py | 22 ++++++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/emcc b/emcc index d1e52f6f8a371..d8d28bcda23fe 100755 --- a/emcc +++ b/emcc @@ -1369,7 +1369,6 @@ try: js_optimizer_extra_info = {} js_optimizer_queue_history = [] js_optimizer_blacklist = (os.environ.get('EMCC_JSOPT_BLACKLIST') or '').split(',') - NATIVE_OPTIMIZER_PASSES = set() # ['optimizeFrounds']) def flush_js_optimizer_queue(title='js_opts'): global final, js_optimizer_queue, js_optimizer_extra_info, js_optimizer_queue_history @@ -1401,7 +1400,7 @@ try: curr = [] native = False for p in passes: - if p not in NATIVE_OPTIMIZER_PASSES: + if p not in shared.js_optimizer.NATIVE_PASSES: if native: chunks.append(['receiveJSON'] + curr + ['emitJSON']) curr = [] diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 2ec01c2d8da74..2af737e0ee202 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -9,6 +9,8 @@ def path_from_root(*pathelems): return os.path.join(__rootpath__, *pathelems) +NATIVE_PASSES = set() # ['optimizeFrounds']) + JS_OPTIMIZER = path_from_root('tools', 'js-optimizer.js') NUM_CHUNKS_PER_CORE = 3 @@ -273,10 +275,22 @@ def write_chunk(chunk, i): filenames = [] if len(filenames) > 0: - # XXX Use '--nocrankshaft' to disable crankshaft to work around v8 bug 1895, needed for older v8/node (node 0.6.8+ should be ok) - commands = map(lambda filename: js_engine + - [JS_OPTIMIZER, filename, 'noPrintMetadata'] + - (['--debug'] if source_map else []) + passes, filenames) + if len(NATIVE_PASSES.intersection(passes)) == 0: + commands = map(lambda filename: js_engine + + [JS_OPTIMIZER, filename, 'noPrintMetadata'] + + (['--debug'] if source_map else []) + passes, filenames) + else: + # use the native optimizer + assert not source_map # XXX need to use js optimizer + def create_optimizer(): + shared.logging.debug('building native optimizer') + output = shared.Cache.get_path('optimizer.exe') + shared.try_delete(output) + subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-I' + shared.path_from_root('tools', 'optimizer', 'rapidjson'), '-std=c++11', '-o', output]).communicate() + assert os.path.exists(output) + return output + native_optimizer = shared.Cache.get('optimizer.exe', create_optimizer, extension='exe') + commands = map(lambda filename: [native_optimizer, filename] + passes, filenames) #print [' '.join(command) for command in commands] cores = min(cores, len(filenames)) From 3320eacb1471d8f8562cc9d7f41eb41096d3d78f Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 30 Oct 2014 16:31:06 -0700 Subject: [PATCH 011/161] fix detection of filename in js optimizer running on chunk --- tools/js_optimizer.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 2af737e0ee202..63c30e17e78a1 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -1,5 +1,5 @@ -import os, sys, subprocess, multiprocessing, re, string, json +import os, sys, subprocess, multiprocessing, re, string, json, shutil import shared configuration = shared.configuration @@ -88,8 +88,13 @@ def serialize(self): def run_on_chunk(command): try: - filename = command[2] # XXX hackish - #print >> sys.stderr, 'running js optimizer command', ' '.join(command), '""""', open(filename).read() + if JS_OPTIMIZER in command: # XXX hackish + index = command.index(JS_OPTIMIZER) + filename = command[index + 1] + else: + filename = command[1] + #print >> sys.stderr, 'running js optimizer command', command, filename + #shutil.copyfile(filename, '/tmp/emscripten_temp/input.txt') output = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0] assert len(output) > 0 and not output.startswith('Assertion failed'), 'Error in js optimizer: ' + output filename = temp_files.get(os.path.basename(filename) + '.jo.js').name @@ -404,6 +409,5 @@ def run(filename, passes, js_engine=shared.NODE_JS, jcache=False, source_map=Fal else: extra_info = None out = run(sys.argv[1], sys.argv[2:], extra_info=extra_info) - import shutil shutil.copyfile(out, sys.argv[1] + '.jsopt.js') From b4838379145741cbfd2b39ba9c832f3f725913b5 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 30 Oct 2014 16:45:32 -0700 Subject: [PATCH 012/161] optimizer fixes --- tools/optimizer/optimizer.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 131ef69051ec0..0a089551315c8 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -37,7 +37,7 @@ GlobalStringValue RETURN("return"), // visit() receives a reference, so it can modify the value being visited directly. // TODO: stoppable version of this void traverseChildren(Value &node, std::function visit) { - assert(node.IsArray()); + if (!node.IsArray()) return; int size = node.Size(); for (int i = 0; i < size; i++) { Value &subnode = node[i]; @@ -60,6 +60,7 @@ Value makeName(const char *str) { //================== bool asm_ = false; +bool preciseF32 = false; //===================== // Optimization passes @@ -111,7 +112,13 @@ int main(int argc, char **argv) { for (int i = 2; i < argc; i++) { std::string str(argv[i]); if (str == "asm") asm_ = true; + else if (str == "asmPreciseF32") preciseF32 = true; + else if (str == "receiveJSON" || str == "emitJSON") {} // the default for us else if (str == "optimizeFrounds") optimizeFrounds(doc); + else { + printf("unrecognized argument: %s\n", str.c_str()); + assert(0); + } } // Emit JSON of modified Document From 525a17d2c07c87f3e6f1722ea340674822c7c00b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 30 Oct 2014 16:57:01 -0700 Subject: [PATCH 013/161] error detection and comments handling in optimizer --- tools/optimizer/optimizer.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 0a089551315c8..d1ddcc434a5fe 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -98,14 +98,20 @@ int main(int argc, char **argv) { int result = stat(input, &st); assert(result == 0); int size = st.st_size; + //printf("reading %s, %d bytes\n", input, size); char *json = new char[size+1]; FILE *f = fopen(input, "rb"); - fread(json, 1, size, f); + int num = fread(json, 1, size, f); + assert(num == size); fclose(f); json[size] = 0; + char *comment = strstr(json, "//"); + if (comment) *comment = 0; // drop off the comments; TODO: parse extra info + // Parse JSON source into a Document doc.Parse(json); + assert(!doc.HasParseError()); delete[] json; // Run passes on the Document From b01e1444876e6c069aea648a0ee1f82d3b84076e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 30 Oct 2014 17:28:47 -0700 Subject: [PATCH 014/161] enable optimizeFrounds as a native optimizer pass --- tools/js_optimizer.py | 4 ++-- tools/optimizer/optimizer.cpp | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 63c30e17e78a1..dc5460212bcf6 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -9,7 +9,7 @@ def path_from_root(*pathelems): return os.path.join(__rootpath__, *pathelems) -NATIVE_PASSES = set() # ['optimizeFrounds']) +NATIVE_PASSES = set(['optimizeFrounds']) JS_OPTIMIZER = path_from_root('tools', 'js-optimizer.js') @@ -291,7 +291,7 @@ def create_optimizer(): shared.logging.debug('building native optimizer') output = shared.Cache.get_path('optimizer.exe') shared.try_delete(output) - subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-I' + shared.path_from_root('tools', 'optimizer', 'rapidjson'), '-std=c++11', '-o', output]).communicate() + subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-I' + shared.path_from_root('tools', 'optimizer', 'rapidjson'), '-O2', '-std=c++11', '-o', output]).communicate() assert os.path.exists(output) return output native_optimizer = shared.Cache.get('optimizer.exe', create_optimizer, extension='exe') diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index d1ddcc434a5fe..17ecddff09b4b 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -33,15 +33,27 @@ GlobalStringValue RETURN("return"), // Infrastructure //================== +void dump(const char *str, Value &node) { + StringBuffer buffer; + Writer writer(buffer); + node.Accept(writer); + printf("%s: ", str); + puts(buffer.GetString()); +} + // Traverses the children of a node. // visit() receives a reference, so it can modify the value being visited directly. // TODO: stoppable version of this void traverseChildren(Value &node, std::function visit) { + //dump("TC", node); if (!node.IsArray()) return; int size = node.Size(); + //printf("size: %d\n", size); for (int i = 0; i < size; i++) { + //printf(" %d:\n", i); Value &subnode = node[i]; - if (subnode.IsArray()) { + if (subnode.IsArray() and subnode.Size() > 0) { + //printf(" go\n"); visit(subnode); } } From 45941ecb0dc2e544114452e4a9671f9b887bc79b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 31 Oct 2014 11:16:40 -0700 Subject: [PATCH 015/161] set up infrastructure to test native optimizer in other.test_js_optimizer --- tests/test_other.py | 14 ++++++++++++++ tools/js_optimizer.py | 20 +++++++++++--------- tools/optimizer/optimizer.cpp | 2 +- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/tests/test_other.py b/tests/test_other.py index 7876ada102fc4..2b27bd5291836 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -1932,8 +1932,22 @@ def test_js_optimizer(self): ['asm', 'ensureLabelSet']), ]: print input + # test calling js optimizer + print ' js' output = Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), input] + passes, stdin=PIPE, stdout=PIPE).communicate()[0] self.assertIdentical(expected, output.replace('\r\n', '\n').replace('\n\n', '\n')) + if 0:#len(js_optimizer.NATIVE_PASSES.intersection(passes)) > 0: + # test calling native + print ' native' + self.clear() + input_temp = 'temp.js' + output_temp = 'output.js' + shutil.copyfile(input, input_temp) + Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), input_temp, 'emitJSON'], stdin=PIPE, stdout=open(input_temp + '.js', 'w')).communicate() + output = Popen([js_optimizer.get_native_optimizer(), input_temp + '.js'] + passes, stdin=PIPE, stdout=open(output_temp, 'w')).communicate()[0] + Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), output_temp, 'emitJSON'], stdin=PIPE, stdout=open(output_temp + '.js', 'w')).communicate() + output = open(output_temp + '.js').read() + self.assertIdentical(expected, output.replace('\r\n', '\n').replace('\n\n', '\n')) def test_m_mm(self): open(os.path.join(self.get_dir(), 'foo.c'), 'w').write('''#include ''') diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index dc5460212bcf6..783b443fff699 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -24,6 +24,16 @@ def path_from_root(*pathelems): func_sig = re.compile('function ([_\w$]+)\(') import_sig = re.compile('var ([_\w$]+) *=[^;]+;') +def get_native_optimizer(): + def create_optimizer(): + shared.logging.debug('building native optimizer') + output = shared.Cache.get_path('optimizer.exe') + shared.try_delete(output) + subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-I' + shared.path_from_root('tools', 'optimizer', 'rapidjson'), '-O2', '-std=c++11', '-o', output]).communicate() + assert os.path.exists(output) + return output + return shared.Cache.get('optimizer.exe', create_optimizer, extension='exe') + class Minifier: ''' asm.js minification support. We calculate minification of @@ -287,15 +297,7 @@ def write_chunk(chunk, i): else: # use the native optimizer assert not source_map # XXX need to use js optimizer - def create_optimizer(): - shared.logging.debug('building native optimizer') - output = shared.Cache.get_path('optimizer.exe') - shared.try_delete(output) - subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-I' + shared.path_from_root('tools', 'optimizer', 'rapidjson'), '-O2', '-std=c++11', '-o', output]).communicate() - assert os.path.exists(output) - return output - native_optimizer = shared.Cache.get('optimizer.exe', create_optimizer, extension='exe') - commands = map(lambda filename: [native_optimizer, filename] + passes, filenames) + commands = map(lambda filename: [get_native_optimizer(), filename] + passes, filenames) #print [' '.join(command) for command in commands] cores = min(cores, len(filenames)) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 17ecddff09b4b..9fa5de6821dbc 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -134,7 +134,7 @@ int main(int argc, char **argv) { else if (str == "receiveJSON" || str == "emitJSON") {} // the default for us else if (str == "optimizeFrounds") optimizeFrounds(doc); else { - printf("unrecognized argument: %s\n", str.c_str()); + fprintf(stderr, "unrecognized argument: %s\n", str.c_str()); assert(0); } } From 0379c53dbc0c0c0690c5f9a486e877ad8f833a96 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 31 Oct 2014 17:52:16 -0700 Subject: [PATCH 016/161] end of the road for rapidjson attempt --- tests/test_other.py | 2 +- tools/js_optimizer.py | 2 +- tools/optimizer/optimizer.cpp | 456 +++++++++++++++++++++++++++++++++- 3 files changed, 453 insertions(+), 7 deletions(-) diff --git a/tests/test_other.py b/tests/test_other.py index 2b27bd5291836..1efebdc48e181 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -1936,7 +1936,7 @@ def test_js_optimizer(self): print ' js' output = Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), input] + passes, stdin=PIPE, stdout=PIPE).communicate()[0] self.assertIdentical(expected, output.replace('\r\n', '\n').replace('\n\n', '\n')) - if 0:#len(js_optimizer.NATIVE_PASSES.intersection(passes)) > 0: + if len(js_optimizer.NATIVE_PASSES.intersection(passes)) > 0: # test calling native print ' native' self.clear() diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 783b443fff699..2a058cfd6b02b 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -9,7 +9,7 @@ def path_from_root(*pathelems): return os.path.join(__rootpath__, *pathelems) -NATIVE_PASSES = set(['optimizeFrounds']) +NATIVE_PASSES = set(['simplifyIfs', 'optimizeFrounds']) JS_OPTIMIZER = path_from_root('tools', 'js-optimizer.js') diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 9fa5de6821dbc..d3db0cd011ae1 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -4,8 +4,11 @@ #include #include +#include +#include #include +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 #include "rapidjson/document.h" #include "rapidjson/writer.h" #include "rapidjson/stringbuffer.h" @@ -24,9 +27,23 @@ class GlobalStringValue : public Value { }; GlobalStringValue RETURN("return"), + STAT("stat"), CALL("call"), NAME("name"), NUM("num"), + SUB("sub"), + ASSIGN("assign"), + UNARY_PREFIX("unary-prefix"), + BINARY("binary"), + CONDITIONAL("conditional"), + SEQ("seq"), + WHILE("while"), + IF("if"), + BLOCK("block"), + LABEL("label"), + TOPLEVEL("toplevel"), + DEFUN("defun"), + BANG("!"), MATH_FROUND("Math_fround"); //================== @@ -37,12 +54,23 @@ void dump(const char *str, Value &node) { StringBuffer buffer; Writer writer(buffer); node.Accept(writer); - printf("%s: ", str); - puts(buffer.GetString()); + fprintf(stderr, "%s: %s\n", str, buffer.GetString()); +} + +int parseInt(const char *str) { + int ret = *str - '0'; + while (*(++str)) { + ret *= 10; + ret += *str - '0'; + } + return ret; } +// Traversals + // Traverses the children of a node. // visit() receives a reference, so it can modify the value being visited directly. +// N.B. we allow replacing arrays in place, and consider strings immutable! // TODO: stoppable version of this void traverseChildren(Value &node, std::function visit) { //dump("TC", node); @@ -59,25 +87,442 @@ void traverseChildren(Value &node, std::function visit) { } } +// Traverse, calling visit before the children +void traversePre(Value &node, std::function visit) { + visit(node); + traverseChildren(node, [&visit](Value& node) { + traversePre(node, visit); + }); +} + +// Traverse, calling visitPre before the children and visitPost after +void traversePrePost(Value &node, std::function visitPre, std::function visitPost) { + visitPre(node); + traverseChildren(node, [&visitPre, &visitPost](Value& node) { + traversePrePost(node, visitPre, visitPost); + }); + visitPost(node); +} + +// Traverses all the top-level functions in the document +void traverseFunctions(Value &ast, std::function visit) { + if (ast[0] == TOPLEVEL) { + Value& stats = ast[1]; + for (int i = 0; i < stats.Size(); i++) { + Value& curr = stats[i]; + if (curr[0] == DEFUN) visit(curr); + } + } else if (ast[0] == DEFUN) { + visit(ast); + } +} + +Value& deStat(Value& node) { + if (node[0] == STAT) return node[1]; + return node; +} + +Value& getStatements(Value& node) { + if (node[0] == DEFUN) { + return node[3]; + } else if (node[0] == BLOCK) { + return node[1]; + } else { + return Value().SetNull(); + } +} + +// Constructions + Value makeName(const char *str) { Value ret; ret.SetArray(); - ret.PushBack(Value().SetString("name", 4), doc.GetAllocator()); + ret.PushBack(NAME, doc.GetAllocator()); ret.PushBack(Value().SetString(str, strlen(str)), doc.GetAllocator()); return ret; } +Value makeNum(double x) { + Value ret; + ret.SetArray(); + ret.PushBack(NUM, doc.GetAllocator()); + ret.PushBack(Value(x), doc.GetAllocator()); + return ret; +} + +Value makeBlock() { + Value ret; + ret.SetArray(); + ret.PushBack(BLOCK, doc.GetAllocator()); + ret.PushBack(Value().SetArray(), doc.GetAllocator()); + return ret; +} + +Value make2(Value& type, Value&& a, Value&& b) { + Value ret; + ret.SetArray(); + ret.PushBack(type, doc.GetAllocator()); + ret.PushBack(a, doc.GetAllocator()); + ret.PushBack(b, doc.GetAllocator()); + return ret; +} +#define mmake2(a, b, c) make2(a, std::move(b), std::move(c)) + +Value make3(Value& type, Value&& a, Value&& b, Value&& c) { + Value ret; + ret.SetArray(); + ret.PushBack(type, doc.GetAllocator()); + ret.PushBack(a, doc.GetAllocator()); + ret.PushBack(b, doc.GetAllocator()); + ret.PushBack(c, doc.GetAllocator()); + return ret; +} +#define mmake3(a, b, c, d) make3(a, std::move(b), std::move(c), std::move(d)) + +// Types + +enum AsmType { + ASM_INT = 0, + ASM_DOUBLE = 1, + ASM_FLOAT = 2, + ASM_FLOAT32X4 = 3, + ASM_INT32X4 = 4, + ASM_NONE = 5 +}; + +struct AsmInfo { + std::map params; // name => index + std::map types; // name => type + + AsmType getType(std::string name) { + auto ret = types.find(name); + if (ret != types.end()) return ret->second; + return ASM_NONE; + } +}; + +struct HeapInfo { + bool valid, unsign, floaty; + int bits; + AsmType type; +}; + +HeapInfo parseHeap(const char *name) { + HeapInfo ret; + if (!strncmp(name, "HEAP", 4)) { + ret.valid = false; + return ret; + } + ret.valid = true; + ret.unsign = name[4] == 'U'; + ret.floaty = name[4] == 'F'; + ret.bits = parseInt(name + (ret.unsign || ret.floaty ? 5 : 4)); + ret.type = !ret.floaty ? ASM_INT : (ret.bits == 64 ? ASM_DOUBLE : ASM_FLOAT); + return ret; +} + +AsmType detectType(Value& node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { + switch (node[0].GetString()[0]) { + case 'n': { + if (node[0] == NUM) { + if (strchr(node[1].GetString(), '.')) return ASM_DOUBLE; + return ASM_INT; + } else if (node[0] == NAME) { + if (asmInfo) { + AsmType ret = asmInfo->getType(node[1].GetString()); + if (ret != ASM_NONE) return ret; + } + if (!inVarDef) { + if (node[1] == "inf" || node[1] == "nan") return ASM_DOUBLE; + if (node[1] == "tempRet0") return ASM_INT; + return ASM_NONE; + } + // We are in a variable definition, where Math_fround(0) optimized into a global constant becomes f0 = Math_fround(0) + assert(0); + /* + if (!ASM_FLOAT_ZERO) ASM_FLOAT_ZERO = node[1]; + else assert(ASM_FLOAT_ZERO == node[1]); + return ASM_FLOAT; + */ + } + break; + } + case 'u': { + if (node[0] == UNARY_PREFIX) { + switch (node[1].GetString()[0]) { + case '+': return ASM_DOUBLE; + case '-': return detectType(node[2], asmInfo, inVarDef); + case '!': case '~': return ASM_INT; + } + break; + } + break; + } + case 'c': { + if (node[0] == CALL) { + if (node[1][0] == NAME) { + Value& name = node[1][1]; + if (name == "Math_fround") return ASM_FLOAT; + else if (name == "SIMD_float32x4") return ASM_FLOAT32X4; + else if (name == "SIMD_int32x4") return ASM_INT32X4; + } + return ASM_NONE; + } else if (node[0] == CONDITIONAL) { + return detectType(node[2], asmInfo, inVarDef); + } + break; + } + case 'b': { + if (node[0] == BINARY) { + switch (node[1].GetString()[0]) { + case '+': case '-': + case '*': case '/': case '%': return detectType(node[2], asmInfo, inVarDef); + case '|': case '&': case '^': case '<': case '>': // handles <<, >>, >>=, <=, >= + case '=': case '!': { // handles ==, != + return ASM_INT; + } + } + } + break; + } + case 's': { + if (node[0] == SEQ) { + return detectType(node[2], asmInfo, inVarDef); + } else if (node[0] == SUB) { + assert(node[1][0] == NAME); + HeapInfo info = parseHeap(node[1][1].GetString()); + if (info.valid) return ASM_NONE; + return info.floaty ? ASM_DOUBLE : ASM_INT; // XXX ASM_FLOAT? + } + break; + } + } + dump("horrible", node); + assert(0); +} + +// Transforms + +// We often have branchings that are simplified so one end vanishes, and +// we then get +// if (!(x < 5)) +// or such. Simplifying these saves space and time. +Value simplifyNotCompsDirect(Value node) { + if (node[0] == UNARY_PREFIX && node[1] == BANG) { + // de-morgan's laws do not work on floats, due to nans >:( + if (node[2][0] == BINARY && (detectType(node[2][2]) == ASM_INT && detectType(node[2][3]) == ASM_INT)) { + Value &op = node[2][1]; + switch(op.GetString()[0]) { + case '<': { + if (op == "<") { op.SetString(">="); break; } + if (op == "<=") { op.SetString(">"); break; } + assert(0); + } + case '>': { + if (op == ">") { op.SetString("<="); break; } + if (op == ">=") { op.SetString("<"); break; } + assert(0); + } + case '=': { + if (op == "==") { op.SetString("!="); break; } + assert(0); + } + case '!': { + if (op == "!=") { op.SetString("=="); break; } + assert(0); + } + default: assert(0); + } + return mmake3(BINARY, op, node[2][2], node[2][3]); + } else if (node[2][0] == UNARY_PREFIX && node[2][1] == '!') { + return std::move(node[2][2]); + } + } + return std::move(node); +} + +Value flipCondition(Value& cond) { + return simplifyNotCompsDirect(mmake2(UNARY_PREFIX, BANG, cond)); +} + +void splice(Value& node, int index) { // removes an element from the middle of an array + node.Erase(node.Begin() + index); +} + +// Checks + +bool commable(Value& node) { // TODO: hashing + std::string type = node[0].GetString(); + if (type == "assign" || type == "binary" || type == "unary-prefix" || type == "unary-postfix" || type == "name" || type == "num" || type == "call" || type == "seq" || type == "conditional" || type == "sub") return true; + return false; +} + //================== // Params //================== -bool asm_ = false; bool preciseF32 = false; //===================== // Optimization passes //===================== +void simplifyIfs(Value& ast) { + traverseFunctions(ast, [](Value& func) { + bool simplifiedAnElse = false; + + traversePre(func, [&simplifiedAnElse](Value& node) { + // simplify if (x) { if (y) { .. } } to if (x ? y : 0) { .. } + if (node[0] == IF) { + Value& body = node[2]; + // recurse to handle chains + while (body[0] == BLOCK) { + Value& stats = body[1]; + if (stats.Size() == 0) break; + Value& other = stats[stats.Size()-1]; + if (other[0] != IF) { + // our if block does not end with an if. perhaps if have an else we can flip + if (node.Size() >= 4 && node[3][0] == BLOCK) { + stats = node[3][1]; + if (stats.Size() == 0) break; + other = stats[stats.Size()-1]; + if (other[0] == IF) { + // flip node + node[1] = flipCondition(node[1]); + node[2] = node[3]; + node[3] = body; + body = node[2]; + } else break; + } else break; + } + // we can handle elses, but must be fully identical + if (node.Size() >= 4 || other.Size() >= 4) { + if (!(node.Size() >= 4)) break; + if (node[3] != other[3]) { + // the elses are different, but perhaps if we flipped a condition we can do better + if (node[3] == other[2]) { + // flip other. note that other may not have had an else! add one if so; we will eliminate such things later + if (!(other.Size() >= 4)) other.PushBack(makeBlock(), doc.GetAllocator()); + other[1] = flipCondition(other[1]); + Value& temp = other[2]; + other[2] = other[3]; + other[3] = temp; + } else break; + } + } + if (stats.Size() > 1) { + // try to commaify - turn everything between the ifs into a comma operator inside the second if + bool ok = true; + for (int i = 0; i < stats.Size()-1; i++) { + Value& curr = deStat(stats[i]); + if (commable(curr)) ok = false; + } + if (!ok) break; + for (int i = stats.Size()-2; i >= 0; i--) { + Value& curr = deStat(stats[i]); + other[1] = mmake2(SEQ, curr, other[1]); + } + Value temp; + temp.SetArray(); + temp.PushBack(other, doc.GetAllocator()); + stats = body[1] = temp; + } + if (stats.Size() != 1) break; + if (node.Size() >= 4) simplifiedAnElse = true; + node[1] = mmake3(CONDITIONAL, node[1], other[1], makeNum(0)); + body = node[2] = other[2]; // XXX + } + } + }); + + if (simplifiedAnElse) { + // there may be fusing opportunities + + // we can only fuse if we remove all uses of the label. if there are + // other ones - if the label check can be reached from elsewhere - + // we must leave it + bool abort = false; + + std::unordered_map labelAssigns; + + traversePre(func, [&labelAssigns, &abort](Value& node) { + if (node[0] == ASSIGN && node[2][0] == NAME && node[2][1] == LABEL) { + if (node[3][0] == NUM) { + std::string value = node[3][1].GetString(); + labelAssigns[value] = labelAssigns[value] + 1; + } else { + // label is assigned a dynamic value (like from indirectbr), we cannot do anything + abort = true; + } + } + }); + if (abort) return; + + std::unordered_map labelChecks; + + traversePre(func, [&labelChecks, &abort](Value& node) { + if (node[0] == BINARY && node[1] == "==" && node[2][0] == BINARY && node[2][1] == '|' && + node[2][2][0] == NAME && node[2][2][1] == LABEL) { + if (node[3][0] == NUM) { + std::string value = node[3][1].GetString(); + labelChecks[value] = labelChecks[value] + 1; + } else { + // label is checked vs a dynamic value (like from indirectbr), we cannot do anything + abort = true; + } + } + }); + if (abort) return; + + int inLoop = 0; // when in a loop, we do not emit label = 0; in the relooper as there is no need + traversePrePost(func, [&inLoop, &labelAssigns, &labelChecks](Value& node) { + if (node[0] == WHILE) inLoop++; + Value& stats = getStatements(node); + if (!stats.IsNull()) { + for (int i = 0; i < stats.Size()-1; i++) { + Value& pre = stats[i]; + Value& post = stats[i+1]; + if (pre[0] == IF && pre.Size() >= 4 && post[0] == IF && post.Size() < 4) { + Value& postCond = post[1]; + if (postCond[0] == BINARY && postCond[1] == "==" && + postCond[2][0] == BINARY && postCond[2][1] == "|" && + postCond[2][2][0] == NAME && postCond[2][2][1] == LABEL && + postCond[2][3][0] == NUM && postCond[2][3][1] == 0 && + postCond[3][0] == NUM) { + Value& postValue = postCond[3][1]; + Value& preElse = pre[3]; + if (labelAssigns[postValue.GetString()] == 1 && labelChecks[postValue.GetString()] == 1 && preElse[0] == BLOCK && preElse.Size() >= 2 && preElse[1].Size() == 1) { + Value& preStat = preElse[1][0]; + if (preStat[0] == STAT && preStat[1][0] == ASSIGN && + preStat[1][1] == true && preStat[1][2][0] == NAME && preStat[1][2][1] == LABEL && + preStat[1][3][0] == NUM && preStat[1][3][1] == postValue) { + // Conditions match, just need to make sure the post clears label + if (post[2][0] == BLOCK && post[2].Size() >= 2 && post[2][1].Size() > 0) { + Value& postStat = post[2][1][0]; + bool haveClear = + postStat[0] == STAT && postStat[1][0] == ASSIGN && + postStat[1][1] == true && postStat[1][2][0] == NAME && postStat[1][2][1] == LABEL && + postStat[1][3][0] == NUM && postStat[1][3][1] == 0; + if (!inLoop || haveClear) { + // Everything lines up, do it + pre[3] = post[2]; + if (haveClear) splice(pre[3][1], 0); // remove the label clearing + splice(stats, i+1); // remove the post entirely + } + } + } + } + } + } + } + } + }, [&inLoop](Value& node) { + if (node[0] == WHILE) inLoop--; + }); + } + }); +} + void optimizeFrounds(Value &ast) { // collapse fround(fround(..)), which can happen due to elimination // also emit f0 instead of fround(0) (except in returns) @@ -129,10 +574,11 @@ int main(int argc, char **argv) { // Run passes on the Document for (int i = 2; i < argc; i++) { std::string str(argv[i]); - if (str == "asm") asm_ = true; + if (str == "asm") {} // the default for us else if (str == "asmPreciseF32") preciseF32 = true; else if (str == "receiveJSON" || str == "emitJSON") {} // the default for us else if (str == "optimizeFrounds") optimizeFrounds(doc); + else if (str == "simplifyIfs") simplifyIfs(doc); else { fprintf(stderr, "unrecognized argument: %s\n", str.c_str()); assert(0); From 336830282bdf2cf6ac24980249fc97cabfe7578c Mon Sep 17 00:00:00 2001 From: Julien Hamaide Date: Tue, 4 Nov 2014 00:18:27 +0100 Subject: [PATCH 017/161] val::__get_handle() is private --- system/include/emscripten/val.h | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/system/include/emscripten/val.h b/system/include/emscripten/val.h index b8a789bc20094..b680dffbc27b1 100644 --- a/system/include/emscripten/val.h +++ b/system/include/emscripten/val.h @@ -7,7 +7,14 @@ namespace emscripten { + + class val; + namespace internal { + + template + val wrapped_extend(const std::string&, const val&); + // Implemented in JavaScript. Don't call these directly. extern "C" { void _emval_register_symbol(const char*); @@ -443,11 +450,6 @@ namespace emscripten { return fromGenericWireType(result); } - // private: TODO: use a friend? - internal::EM_VAL __get_handle() const { - return handle; - } - val typeof() const { return val(_emval_typeof(handle)); } @@ -458,6 +460,13 @@ namespace emscripten { : handle(handle) {} + template + friend val internal::wrapped_extend(const std::string& , const val& ); + + internal::EM_VAL __get_handle() const { + return handle; + } + internal::EM_VAL handle; friend struct internal::BindingType; From befcd58a309867d6c127a8b154780c94e9078706 Mon Sep 17 00:00:00 2001 From: Julien Hamaide Date: Tue, 4 Nov 2014 00:29:43 +0100 Subject: [PATCH 018/161] Factorize val::call and val::new --- system/include/emscripten/val.h | 40 ++++++++++++++------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/system/include/emscripten/val.h b/system/include/emscripten/val.h index b680dffbc27b1..8f2a2f5d21174 100644 --- a/system/include/emscripten/val.h +++ b/system/include/emscripten/val.h @@ -385,20 +385,9 @@ namespace emscripten { template val new_(Args&&... args) const { - using namespace internal; - - WithPolicies<>::ArgTypeList argList; - WireTypePack argv(std::forward(args)...); - // todo: this is awfully similar to operator(), can we - // merge them somehow? - return val( - _emval_new( - handle, - argList.getCount(), - argList.getTypes(), - argv)); + return internalCall(internal::_emval_new,std::forward(args)...); } - + template val operator[](const T& key) const { return val(internal::_emval_get_property(handle, val(key).handle)); @@ -416,16 +405,7 @@ namespace emscripten { template val operator()(Args&&... args) { - using namespace internal; - - WithPolicies<>::ArgTypeList argList; - WireTypePack argv(std::forward(args)...); - return val( - _emval_call( - handle, - argList.getCount(), - argList.getTypes(), - argv)); + return internalCall(internal::_emval_call, std::forward(args)...); } template @@ -467,6 +447,20 @@ namespace emscripten { return handle; } + template + val internalCall(Implementation impl, Args&&... args)const { + using namespace internal; + + WithPolicies<>::ArgTypeList argList; + WireTypePack argv(std::forward(args)...); + return val( + impl( + handle, + argList.getCount(), + argList.getTypes(), + argv)); + } + internal::EM_VAL handle; friend struct internal::BindingType; From e90043c74127bd28c856de9bdb7e00ffb8bd536c Mon Sep 17 00:00:00 2001 From: Julien Hamaide Date: Tue, 4 Nov 2014 00:58:30 +0100 Subject: [PATCH 019/161] val::global() returns the global object --- src/embind/emval.js | 8 ++++++-- system/include/emscripten/val.h | 2 +- tests/embind/embind.test.js | 4 ++++ tests/embind/embind_test.cpp | 6 ++++++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/embind/emval.js b/src/embind/emval.js index 84af9870f4e44..51ad4ce97e1d9 100644 --- a/src/embind/emval.js +++ b/src/embind/emval.js @@ -188,8 +188,12 @@ var LibraryEmVal = { $emval_get_global: function() { return (function(){return Function;})()('return this')(); }, _emval_get_global__deps: ['_emval_register', '$getStringOrSymbol', '$emval_get_global'], _emval_get_global: function(name) { - name = getStringOrSymbol(name); - return __emval_register(emval_get_global()[name]); + if(name===0){ + return __emval_register(emval_get_global()); + } else { + name = getStringOrSymbol(name); + return __emval_register(emval_get_global()[name]); + } }, _emval_get_module_property__deps: ['$getStringOrSymbol', '_emval_register'], diff --git a/system/include/emscripten/val.h b/system/include/emscripten/val.h index 8f2a2f5d21174..d83a7d1f9ae02 100644 --- a/system/include/emscripten/val.h +++ b/system/include/emscripten/val.h @@ -300,7 +300,7 @@ namespace emscripten { return val(e); } - static val global(const char* name) { + static val global(const char* name = 0) { return val(internal::_emval_get_global(name)); } diff --git a/tests/embind/embind.test.js b/tests/embind/embind.test.js index 192e2e904cd44..be74e53628ab4 100644 --- a/tests/embind/embind.test.js +++ b/tests/embind/embind.test.js @@ -581,6 +581,10 @@ module({ cm.emval_test_take_and_return_std_string_const_ref("foobar"); }); + test("can get global", function(){ + assert.equal((new Function("return this;"))(), cm.embind_test_getglobal()); + }); + test("can create new object", function() { assert.deepEqual({}, cm.embind_test_new_Object()); }); diff --git a/tests/embind/embind_test.cpp b/tests/embind/embind_test.cpp index c274a0f111f89..b589d13fec0fe 100644 --- a/tests/embind/embind_test.cpp +++ b/tests/embind/embind_test.cpp @@ -1122,6 +1122,10 @@ void test_string_with_vec(const std::string& p1, std::vector& v1) { printf("%s\n", p1.c_str()); } +val embind_test_getglobal() { + return val::global(); +} + val embind_test_new_Object() { return val::global("Object").new_(); } @@ -1981,6 +1985,8 @@ EMSCRIPTEN_BINDINGS(tests) { register_map("StringIntMap"); function("embind_test_get_string_int_map", embind_test_get_string_int_map); + function("embind_test_getglobal", &embind_test_getglobal); + function("embind_test_new_Object", &embind_test_new_Object); function("embind_test_new_factory", &embind_test_new_factory); From 2076afb706c6dfc80f6a61cef54b7fbf3189551d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 3 Nov 2014 16:09:28 -0800 Subject: [PATCH 020/161] minijson.h --- tools/optimizer/minijson.h | 217 +++++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 tools/optimizer/minijson.h diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h new file mode 100644 index 0000000000000..7f66ae77d95d4 --- /dev/null +++ b/tools/optimizer/minijson.h @@ -0,0 +1,217 @@ +/* + * Extremely minimal JSON, just enough to manipulate an AST in the format the JS optimizer wants: arrays, strings, numbers, bools (no objects, no non-ascii, etc.). + * Optimized for fast parsing and also manipulation of the AST. + * Uses shared_ptr for simplicity, basically everywhere. TODO: measure impact + */ + +#include +#include +#include +#include + +#include +#include +#include + +struct Value; + +typedef std::shared_ptr Ref; +typedef std::vector ArrayStorage; + +struct Value { + enum Type { + String = 0, + Number = 1, + Array = 2, + Null = 3, + Bool = 4 + }; + + Type type; + + union { + std::string *str; + double num; + ArrayStorage *arr; + bool boo; + }; + + // constructors all copy their input + Value() : type(Null), num(0) {} + Value(const char *s) : type(Null) { + set(s); + } + Value(double n) : type(Null) { + set(n); + } + Value(ArrayStorage &a) : type(Null) { + setArray(); + *arr = a; + } + Value(bool b) : type(Null) { + setBool(b); + } + + ~Value() { + free(); + } + + void free() { + if (type == String) delete str; + else if (type == Array) delete arr; + type = Null; + num = 0; + } + + void set(const char *s) { + free(); + type = String; + str = new std::string(s); + } + void set(double n) { + free(); + type = Null; + num = n; + } + void set(ArrayStorage &a) { + free(); + type = Array; + arr = new ArrayStorage(); + *arr = a; + } + void setArray() { + free(); + type = Array; + arr = new ArrayStorage(); + } + void setNull() { + free(); + type = Null; + } + void setBool(bool b) { + free(); + type = Bool; + boo = b; + } + + Value& operator=(const Value& other) { + free(); + switch (other.type) { + case String: + set(other.str->c_str()); + break; + case Number: + set(other.num); + break; + case Array: + set(*other.arr); + break; + case Null: + setNull(); + break; + case Bool: + setBool(other.boo); + break; + } + return *this; + } + + bool operator==(const Value& other) { + if (type != other.type) return false; + switch (other.type) { + case String: + return *str == *(other.str); + case Number: + return num == other.num; + case Array: + if (arr->size() != other.arr->size()) return false; + for (unsigned i = 0; i < arr->size(); i++) { + if ((*arr)[i] != (*other.arr)[i]) return false; + } + break; + case Null: + break; + case Bool: + return boo == other.boo; + } + return true; + } + + char* parse(char* curr) { + #define skip() { while (*curr && isspace(*curr)) curr++; } + skip(); + if (*curr == '"') { + // String + curr++; + char *close = strchr(curr, '"'); + assert(close); + *close = 0; + set(curr); + *close = '"'; + curr = close+1; + } else if (*curr == '[') { + // Array + curr++; + skip(); + setArray(); + while (*curr != ']') { + Ref temp(new Value); + arr->push_back(temp); + curr = temp->parse(curr); + skip(); + if (*curr == ']') break; + assert(*curr == ','); + curr++; + skip(); + } + curr++; + } else if (*curr == 'n') { + // Null + assert(strncmp(curr, "null", 4) == 0); + setNull(); + curr += 4; + } else if (*curr == 't') { + // Bool true + assert(strncmp(curr, "true", 4) == 0); + setBool(true); + curr += 4; + } else if (*curr == 'f') { + // Bool false + assert(strncmp(curr, "false", 5) == 0); + setBool(false); + curr += 5; + } else { + // Number + char *after; + num = strtod(curr, &after); + curr = after; + } + return curr; + } + + void stringify(std::ostream &os) { + switch (type) { + case String: + os << '"' << *str << '"'; + break; + case Number: + os << num; + break; + case Array: + os << '['; + for (unsigned i = 0; i < arr->size(); i++) { + if (i > 0) os << ", "; + (*arr)[i]->stringify(os); + } + os << ']'; + break; + case Null: + os << "null"; + break; + case Bool: + os << (boo ? "true" : "false"); + break; + } + } +}; + From 048cc5b9b9c5bc0016c4e81db037fe37db08a318 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 3 Nov 2014 17:32:38 -0800 Subject: [PATCH 021/161] get most of optimizer building on new minijson.h --- tools/optimizer/minijson.h | 115 ++++++++-- tools/optimizer/optimizer.cpp | 388 ++++++++++++++++------------------ 2 files changed, 279 insertions(+), 224 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 7f66ae77d95d4..9393e9963b787 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -15,9 +15,25 @@ struct Value; -typedef std::shared_ptr Ref; +// Reference to a value. Simple shared_ptr, plus [] operator for convenience - we work on lots of arrays +class Ref : public std::shared_ptr { +public: + Ref(Value *v) { + reset(v); + } + + Ref& operator[](unsigned x) { + return (*this)[x]; + } + + // special convenience for comparison to string, which is by value + bool operator==(const char *str); + bool operator!=(const char *str); +}; + typedef std::vector ArrayStorage; +// Main value type struct Value { enum Type { String = 0, @@ -29,7 +45,7 @@ struct Value { Type type; - union { + union { // TODO: optimize std::string *str; double num; ArrayStorage *arr; @@ -38,19 +54,17 @@ struct Value { // constructors all copy their input Value() : type(Null), num(0) {} - Value(const char *s) : type(Null) { + explicit Value(const char *s) : type(Null) { set(s); } - Value(double n) : type(Null) { + explicit Value(double n) : type(Null) { set(n); } - Value(ArrayStorage &a) : type(Null) { + explicit Value(ArrayStorage &a) : type(Null) { setArray(); *arr = a; } - Value(bool b) : type(Null) { - setBool(b); - } + // no bool constructor - would endanger the double one (int might convert the wrong way) ~Value() { free(); @@ -63,35 +77,66 @@ struct Value { num = 0; } - void set(const char *s) { + Value& set(const char *s) { free(); type = String; str = new std::string(s); + return *this; } - void set(double n) { + Value& set(double n) { free(); type = Null; num = n; + return *this; } - void set(ArrayStorage &a) { + Value& set(ArrayStorage &a) { free(); type = Array; arr = new ArrayStorage(); *arr = a; + return *this; } - void setArray() { + Value& setArray() { free(); type = Array; arr = new ArrayStorage(); + return *this; } - void setNull() { + Value& setNull() { free(); type = Null; + return *this; } - void setBool(bool b) { + Value& setBool(bool b) { // Bool in the name, as otherwise might overload over int free(); type = Bool; boo = b; + return *this; + } + + bool isString() { return type == String; } + bool isNumber() { return type == Number; } + bool isArray() { return type == Array; } + bool isNull() { return type == Null; } + bool isBool() { return type == Bool; } + + bool isBool(bool b) { return type == Bool && b == boo; } // avoid overloading == as it might overload over int + + std::string& getString() { + assert(isString()); + return *str; + } + const char* getCString() { + assert(isString()); + return str->c_str(); + } + double& getNumber() { + assert(isNumber()); + return num; + } + bool& getBool() { + assert(isBool()); + return boo; } Value& operator=(const Value& other) { @@ -213,5 +258,47 @@ struct Value { break; } } + + // String operations + + // Number operations + + // Array operations + + unsigned size() { + assert(isArray()); + return arr->size(); + } + + Ref& operator[](unsigned x) { + assert(isArray()); + return (*arr)[x]; + } + + void push_back(Ref r) { + assert(isArray()); + arr->push_back(r); + } + + // Null operations + + // Bool operations }; +// Convenience class to construct arrays +struct ArrayValue : public Value { + ArrayValue() { + setArray(); + } +}; + +// + +bool Ref::operator==(const char *str) { + return get()->isString() && get()->getString() == str; +} + +bool Ref::operator!=(const char *str) { + return get()->isString() ? get()->getString() != str : true; +} + diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index d3db0cd011ae1..bb45e34b99585 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -7,54 +7,23 @@ #include #include #include +#include -#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 -#include "rapidjson/document.h" -#include "rapidjson/writer.h" -#include "rapidjson/stringbuffer.h" - -using namespace rapidjson; +#include "minijson.h" //================== // Globals //================== -Document doc; - -class GlobalStringValue : public Value { -public: - GlobalStringValue(const char *str) : Value(str, strlen(str)) {} -}; - -GlobalStringValue RETURN("return"), - STAT("stat"), - CALL("call"), - NAME("name"), - NUM("num"), - SUB("sub"), - ASSIGN("assign"), - UNARY_PREFIX("unary-prefix"), - BINARY("binary"), - CONDITIONAL("conditional"), - SEQ("seq"), - WHILE("while"), - IF("if"), - BLOCK("block"), - LABEL("label"), - TOPLEVEL("toplevel"), - DEFUN("defun"), - BANG("!"), - MATH_FROUND("Math_fround"); +Value doc; //================== // Infrastructure //================== -void dump(const char *str, Value &node) { - StringBuffer buffer; - Writer writer(buffer); - node.Accept(writer); - fprintf(stderr, "%s: %s\n", str, buffer.GetString()); +void dump(const char *str, Ref node) { + std::cerr << str << ": "; + node->stringify(std::cerr); } int parseInt(const char *str) { @@ -72,15 +41,15 @@ int parseInt(const char *str) { // visit() receives a reference, so it can modify the value being visited directly. // N.B. we allow replacing arrays in place, and consider strings immutable! // TODO: stoppable version of this -void traverseChildren(Value &node, std::function visit) { +void traverseChildren(Ref node, std::function visit) { //dump("TC", node); - if (!node.IsArray()) return; - int size = node.Size(); + if (!node->isArray()) return; + int size = node->size(); //printf("size: %d\n", size); for (int i = 0; i < size; i++) { //printf(" %d:\n", i); - Value &subnode = node[i]; - if (subnode.IsArray() and subnode.Size() > 0) { + Ref subnode = node[i]; + if (subnode->isArray() and subnode->size() > 0) { //printf(" go\n"); visit(subnode); } @@ -88,96 +57,97 @@ void traverseChildren(Value &node, std::function visit) { } // Traverse, calling visit before the children -void traversePre(Value &node, std::function visit) { +void traversePre(Ref node, std::function visit) { visit(node); - traverseChildren(node, [&visit](Value& node) { + traverseChildren(node, [&visit](Ref node) { traversePre(node, visit); }); } // Traverse, calling visitPre before the children and visitPost after -void traversePrePost(Value &node, std::function visitPre, std::function visitPost) { +void traversePrePost(Ref node, std::function visitPre, std::function visitPost) { visitPre(node); - traverseChildren(node, [&visitPre, &visitPost](Value& node) { + traverseChildren(node, [&visitPre, &visitPost](Ref node) { traversePrePost(node, visitPre, visitPost); }); visitPost(node); } // Traverses all the top-level functions in the document -void traverseFunctions(Value &ast, std::function visit) { - if (ast[0] == TOPLEVEL) { - Value& stats = ast[1]; - for (int i = 0; i < stats.Size(); i++) { - Value& curr = stats[i]; - if (curr[0] == DEFUN) visit(curr); +void traverseFunctions(Ref ast, std::function visit) { + if (ast[0] == "toplevel") { + Ref stats = ast[1]; + for (int i = 0; i < stats->size(); i++) { + Ref curr = stats[i]; + if (curr[0] == "defun") visit(curr); } - } else if (ast[0] == DEFUN) { + } else if (ast[0] == "defun") { visit(ast); } } -Value& deStat(Value& node) { - if (node[0] == STAT) return node[1]; +Ref deStat(Ref node) { + if (node[0]->getString() =="stat") return node[1]; return node; } -Value& getStatements(Value& node) { - if (node[0] == DEFUN) { +Ref getStatements(Ref node) { + if (node[0] == "defun") { return node[3]; - } else if (node[0] == BLOCK) { + } else if (node[0] == "block") { return node[1]; } else { - return Value().SetNull(); + return new Value(); } } -// Constructions +// Constructions TODO: share common constructions, and assert they remain frozen + +Ref makeName(const char *str) { + Ref ret(new ArrayValue()); + ret->push_back(new Value("name")); + ret->push_back(new Value(str)); + return ret; +} -Value makeName(const char *str) { - Value ret; - ret.SetArray(); - ret.PushBack(NAME, doc.GetAllocator()); - ret.PushBack(Value().SetString(str, strlen(str)), doc.GetAllocator()); +Ref makeNum(double x) { + Ref ret(new ArrayValue()); + ret->push_back(new Value("num")); + ret->push_back(new Value(x)); return ret; } -Value makeNum(double x) { - Value ret; - ret.SetArray(); - ret.PushBack(NUM, doc.GetAllocator()); - ret.PushBack(Value(x), doc.GetAllocator()); +Ref makeBlock() { + Ref ret(new ArrayValue()); + ret->push_back(new Value("block")); + ret->push_back(new ArrayValue()); return ret; } -Value makeBlock() { - Value ret; - ret.SetArray(); - ret.PushBack(BLOCK, doc.GetAllocator()); - ret.PushBack(Value().SetArray(), doc.GetAllocator()); +Ref make2(const char* type, const char *a, Ref b) { + Ref ret(new ArrayValue()); + ret->push_back(new Value(type)); + ret->push_back(new Value(a)); + ret->push_back(b); return ret; } -Value make2(Value& type, Value&& a, Value&& b) { - Value ret; - ret.SetArray(); - ret.PushBack(type, doc.GetAllocator()); - ret.PushBack(a, doc.GetAllocator()); - ret.PushBack(b, doc.GetAllocator()); +Ref make2(const char* type, Ref a, Ref b) { + Ref ret(new ArrayValue()); + ret->push_back(new Value(type)); + ret->push_back(a); + ret->push_back(b); return ret; } -#define mmake2(a, b, c) make2(a, std::move(b), std::move(c)) - -Value make3(Value& type, Value&& a, Value&& b, Value&& c) { - Value ret; - ret.SetArray(); - ret.PushBack(type, doc.GetAllocator()); - ret.PushBack(a, doc.GetAllocator()); - ret.PushBack(b, doc.GetAllocator()); - ret.PushBack(c, doc.GetAllocator()); + +Ref make3(const char *type, Ref a, Ref b, Ref c) { + Ref ret(new ArrayValue()); + ret->push_back(new Value(type)); + ret->push_back(a); + ret->push_back(b); + ret->push_back(c); return ret; } -#define mmake3(a, b, c, d) make3(a, std::move(b), std::move(c), std::move(d)) // Types @@ -221,15 +191,15 @@ HeapInfo parseHeap(const char *name) { return ret; } -AsmType detectType(Value& node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { - switch (node[0].GetString()[0]) { +AsmType detectType(Ref node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { + switch (node[0]->getString()[0]) { case 'n': { - if (node[0] == NUM) { - if (strchr(node[1].GetString(), '.')) return ASM_DOUBLE; + if (node[0] == "num") { + if (strchr(node[1]->getCString(), '.')) return ASM_DOUBLE; return ASM_INT; - } else if (node[0] == NAME) { + } else if (node[0] == "name") { if (asmInfo) { - AsmType ret = asmInfo->getType(node[1].GetString()); + AsmType ret = asmInfo->getType(node[1]->getString()); if (ret != ASM_NONE) return ret; } if (!inVarDef) { @@ -248,8 +218,8 @@ AsmType detectType(Value& node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { break; } case 'u': { - if (node[0] == UNARY_PREFIX) { - switch (node[1].GetString()[0]) { + if (node[0] == "unary-prefix") { + switch (node[1]->getString()[0]) { case '+': return ASM_DOUBLE; case '-': return detectType(node[2], asmInfo, inVarDef); case '!': case '~': return ASM_INT; @@ -259,22 +229,22 @@ AsmType detectType(Value& node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { break; } case 'c': { - if (node[0] == CALL) { - if (node[1][0] == NAME) { - Value& name = node[1][1]; + if (node[0] == "call") { + if (node[1][0] == "name") { + std::string& name = node[1][1]->getString(); if (name == "Math_fround") return ASM_FLOAT; else if (name == "SIMD_float32x4") return ASM_FLOAT32X4; else if (name == "SIMD_int32x4") return ASM_INT32X4; } return ASM_NONE; - } else if (node[0] == CONDITIONAL) { + } else if (node[0] == "conditional") { return detectType(node[2], asmInfo, inVarDef); } break; } case 'b': { - if (node[0] == BINARY) { - switch (node[1].GetString()[0]) { + if (node[0] == "binary") { + switch (node[1]->getString()[0]) { case '+': case '-': case '*': case '/': case '%': return detectType(node[2], asmInfo, inVarDef); case '|': case '&': case '^': case '<': case '>': // handles <<, >>, >>=, <=, >= @@ -286,11 +256,11 @@ AsmType detectType(Value& node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { break; } case 's': { - if (node[0] == SEQ) { + if (node[0] == "seq") { return detectType(node[2], asmInfo, inVarDef); - } else if (node[0] == SUB) { - assert(node[1][0] == NAME); - HeapInfo info = parseHeap(node[1][1].GetString()); + } else if (node[0] == "sub") { + assert(node[1][0] == "name"); + HeapInfo info = parseHeap(node[1][1]->getCString()); if (info.valid) return ASM_NONE; return info.floaty ? ASM_DOUBLE : ASM_INT; // XXX ASM_FLOAT? } @@ -307,52 +277,52 @@ AsmType detectType(Value& node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { // we then get // if (!(x < 5)) // or such. Simplifying these saves space and time. -Value simplifyNotCompsDirect(Value node) { - if (node[0] == UNARY_PREFIX && node[1] == BANG) { +Ref simplifyNotCompsDirect(Ref node) { + if (node[0] == "unary-prefix" && node[1] == "!") { // de-morgan's laws do not work on floats, due to nans >:( - if (node[2][0] == BINARY && (detectType(node[2][2]) == ASM_INT && detectType(node[2][3]) == ASM_INT)) { - Value &op = node[2][1]; - switch(op.GetString()[0]) { + if (node[2][0] == "binary" && (detectType(node[2][2]) == ASM_INT && detectType(node[2][3]) == ASM_INT)) { + Ref op = node[2][1]; + switch(op->getCString()[0]) { case '<': { - if (op == "<") { op.SetString(">="); break; } - if (op == "<=") { op.SetString(">"); break; } + if (op == "<") { op->set(">="); break; } + if (op == "<=") { op->set(">"); break; } assert(0); } case '>': { - if (op == ">") { op.SetString("<="); break; } - if (op == ">=") { op.SetString("<"); break; } + if (op == ">") { op->set("<="); break; } + if (op == ">=") { op->set("<"); break; } assert(0); } case '=': { - if (op == "==") { op.SetString("!="); break; } + if (op == "==") { op->set("!="); break; } assert(0); } case '!': { - if (op == "!=") { op.SetString("=="); break; } + if (op == "!=") { op->set("=="); break; } assert(0); } default: assert(0); } - return mmake3(BINARY, op, node[2][2], node[2][3]); - } else if (node[2][0] == UNARY_PREFIX && node[2][1] == '!') { - return std::move(node[2][2]); + return make3("binary", op, node[2][2], node[2][3]); + } else if (node[2][0] == "unary-prefix" && node[2][1] == "!") { + return node[2][2]; } } - return std::move(node); + return node; } -Value flipCondition(Value& cond) { - return simplifyNotCompsDirect(mmake2(UNARY_PREFIX, BANG, cond)); +Ref flipCondition(Ref cond) { + return simplifyNotCompsDirect(make2("unary-prefix", "!", cond)); } -void splice(Value& node, int index) { // removes an element from the middle of an array - node.Erase(node.Begin() + index); +void splice(Ref node, int index) { // removes an element from the middle of an array +assert(0);// node.Erase(node.Begin() + index); } // Checks -bool commable(Value& node) { // TODO: hashing - std::string type = node[0].GetString(); +bool commable(Ref node) { // TODO: hashing + std::string type = node[0]->getString(); if (type == "assign" || type == "binary" || type == "unary-prefix" || type == "unary-postfix" || type == "name" || type == "num" || type == "call" || type == "seq" || type == "conditional" || type == "sub") return true; return false; } @@ -367,26 +337,26 @@ bool preciseF32 = false; // Optimization passes //===================== -void simplifyIfs(Value& ast) { - traverseFunctions(ast, [](Value& func) { +void simplifyIfs(Ref ast) { + traverseFunctions(ast, [](Ref func) { bool simplifiedAnElse = false; - traversePre(func, [&simplifiedAnElse](Value& node) { + traversePre(func, [&simplifiedAnElse](Ref node) { // simplify if (x) { if (y) { .. } } to if (x ? y : 0) { .. } - if (node[0] == IF) { - Value& body = node[2]; + if (node[0] == "if") { + Ref body = node[2]; // recurse to handle chains - while (body[0] == BLOCK) { - Value& stats = body[1]; - if (stats.Size() == 0) break; - Value& other = stats[stats.Size()-1]; - if (other[0] != IF) { + while (body[0] == "block") { + Ref stats = body[1]; + if (stats->size() == 0) break; + Ref other = stats[stats->size()-1]; + if (other[0] != "if") { // our if block does not end with an if. perhaps if have an else we can flip - if (node.Size() >= 4 && node[3][0] == BLOCK) { + if (node->size() >= 4 && node[3][0] == "block") { stats = node[3][1]; - if (stats.Size() == 0) break; - other = stats[stats.Size()-1]; - if (other[0] == IF) { + if (stats->size() == 0) break; + other = stats[stats->size()-1]; + if (other[0] == "if") { // flip node node[1] = flipCondition(node[1]); node[2] = node[3]; @@ -396,41 +366,40 @@ void simplifyIfs(Value& ast) { } else break; } // we can handle elses, but must be fully identical - if (node.Size() >= 4 || other.Size() >= 4) { - if (!(node.Size() >= 4)) break; + if (node->size() >= 4 || other->size() >= 4) { + if (!(node->size() >= 4)) break; if (node[3] != other[3]) { // the elses are different, but perhaps if we flipped a condition we can do better if (node[3] == other[2]) { // flip other. note that other may not have had an else! add one if so; we will eliminate such things later - if (!(other.Size() >= 4)) other.PushBack(makeBlock(), doc.GetAllocator()); + if (!(other->size() >= 4)) other->push_back(makeBlock()); other[1] = flipCondition(other[1]); - Value& temp = other[2]; + Ref temp = other[2]; other[2] = other[3]; other[3] = temp; } else break; } } - if (stats.Size() > 1) { + if (stats->size() > 1) { // try to commaify - turn everything between the ifs into a comma operator inside the second if bool ok = true; - for (int i = 0; i < stats.Size()-1; i++) { - Value& curr = deStat(stats[i]); + for (int i = 0; i < stats->size()-1; i++) { + Ref curr = deStat(stats[i]); if (commable(curr)) ok = false; } if (!ok) break; - for (int i = stats.Size()-2; i >= 0; i--) { - Value& curr = deStat(stats[i]); - other[1] = mmake2(SEQ, curr, other[1]); + for (int i = stats->size()-2; i >= 0; i--) { + Ref curr = deStat(stats[i]); + other[1] = make2("seq", curr, other[1]); } - Value temp; - temp.SetArray(); - temp.PushBack(other, doc.GetAllocator()); + Ref temp = new ArrayValue(); + temp->push_back(other); stats = body[1] = temp; } - if (stats.Size() != 1) break; - if (node.Size() >= 4) simplifiedAnElse = true; - node[1] = mmake3(CONDITIONAL, node[1], other[1], makeNum(0)); - body = node[2] = other[2]; // XXX + if (stats->size() != 1) break; + if (node->size() >= 4) simplifiedAnElse = true; + node[1] = make3("conditional", node[1], other[1], makeNum(0)); + body = node[2] = other[2]; } } }); @@ -445,10 +414,10 @@ void simplifyIfs(Value& ast) { std::unordered_map labelAssigns; - traversePre(func, [&labelAssigns, &abort](Value& node) { - if (node[0] == ASSIGN && node[2][0] == NAME && node[2][1] == LABEL) { - if (node[3][0] == NUM) { - std::string value = node[3][1].GetString(); + traversePre(func, [&labelAssigns, &abort](Ref node) { + if (node[0] == "assign" && node[2][0] == "name" && node[2][1] == "label") { + if (node[3][0] == "num") { + std::string value = node[3][1]->getString(); labelAssigns[value] = labelAssigns[value] + 1; } else { // label is assigned a dynamic value (like from indirectbr), we cannot do anything @@ -460,11 +429,11 @@ void simplifyIfs(Value& ast) { std::unordered_map labelChecks; - traversePre(func, [&labelChecks, &abort](Value& node) { - if (node[0] == BINARY && node[1] == "==" && node[2][0] == BINARY && node[2][1] == '|' && - node[2][2][0] == NAME && node[2][2][1] == LABEL) { - if (node[3][0] == NUM) { - std::string value = node[3][1].GetString(); + traversePre(func, [&labelChecks, &abort](Ref node) { + if (node[0] == "binary" && node[1] == "==" && node[2][0] == "binary" && node[2][1] == "|" && + node[2][2][0] == "name" && node[2][2][1] == "label") { + if (node[3][0] == "num") { + std::string value = node[3][1]->getString(); labelChecks[value] = labelChecks[value] + 1; } else { // label is checked vs a dynamic value (like from indirectbr), we cannot do anything @@ -475,34 +444,34 @@ void simplifyIfs(Value& ast) { if (abort) return; int inLoop = 0; // when in a loop, we do not emit label = 0; in the relooper as there is no need - traversePrePost(func, [&inLoop, &labelAssigns, &labelChecks](Value& node) { - if (node[0] == WHILE) inLoop++; - Value& stats = getStatements(node); - if (!stats.IsNull()) { - for (int i = 0; i < stats.Size()-1; i++) { - Value& pre = stats[i]; - Value& post = stats[i+1]; - if (pre[0] == IF && pre.Size() >= 4 && post[0] == IF && post.Size() < 4) { - Value& postCond = post[1]; - if (postCond[0] == BINARY && postCond[1] == "==" && - postCond[2][0] == BINARY && postCond[2][1] == "|" && - postCond[2][2][0] == NAME && postCond[2][2][1] == LABEL && - postCond[2][3][0] == NUM && postCond[2][3][1] == 0 && - postCond[3][0] == NUM) { - Value& postValue = postCond[3][1]; - Value& preElse = pre[3]; - if (labelAssigns[postValue.GetString()] == 1 && labelChecks[postValue.GetString()] == 1 && preElse[0] == BLOCK && preElse.Size() >= 2 && preElse[1].Size() == 1) { - Value& preStat = preElse[1][0]; - if (preStat[0] == STAT && preStat[1][0] == ASSIGN && - preStat[1][1] == true && preStat[1][2][0] == NAME && preStat[1][2][1] == LABEL && - preStat[1][3][0] == NUM && preStat[1][3][1] == postValue) { + traversePrePost(func, [&inLoop, &labelAssigns, &labelChecks](Ref node) { + if (node[0] == "while") inLoop++; + Ref stats = getStatements(node); + if (!stats->isNull()) { + for (int i = 0; i < stats->size()-1; i++) { + Ref pre = stats[i]; + Ref post = stats[i+1]; + if (pre[0] == "if" && pre->size() >= 4 && post[0] == "if" && post->size() < 4) { + Ref postCond = post[1]; + if (postCond[0] == "binary" && postCond[1] == "==" && + postCond[2][0] == "binary" && postCond[2][1] == "|" && + postCond[2][2][0] == "name" && postCond[2][2][1] == "label" && + postCond[2][3][0] == "num" && postCond[2][3][1] == 0 && + postCond[3][0] == "num") { + Ref postValue = postCond[3][1]; + Ref preElse = pre[3]; + if (labelAssigns[postValue->getString()] == 1 && labelChecks[postValue->getString()] == 1 && preElse[0] == "block" && preElse->size() >= 2 && preElse[1]->size() == 1) { + Ref preStat = preElse[1][0]; + if (preStat[0] == "stat" && preStat[1][0] == "assign" && + preStat[1][1]->isBool(true) && preStat[1][2][0] == "name" && preStat[1][2][1] == "label" && + preStat[1][3][0] == "num" && preStat[1][3][1] == postValue) { // Conditions match, just need to make sure the post clears label - if (post[2][0] == BLOCK && post[2].Size() >= 2 && post[2][1].Size() > 0) { - Value& postStat = post[2][1][0]; + if (post[2][0] == "block" && post[2]->size() >= 2 && post[2][1]->size() > 0) { + Ref postStat = post[2][1][0]; bool haveClear = - postStat[0] == STAT && postStat[1][0] == ASSIGN && - postStat[1][1] == true && postStat[1][2][0] == NAME && postStat[1][2][1] == LABEL && - postStat[1][3][0] == NUM && postStat[1][3][1] == 0; + postStat[0] == "stat" && postStat[1][0] == "assign" && + postStat[1][1]->isBool(true) && postStat[1][2][0] == "name" && postStat[1][2][1] == "label" && + postStat[1][3][0] == "num" && postStat[1][3][1] == 0; if (!inLoop || haveClear) { // Everything lines up, do it pre[3] = post[2]; @@ -516,27 +485,27 @@ void simplifyIfs(Value& ast) { } } } - }, [&inLoop](Value& node) { - if (node[0] == WHILE) inLoop--; + }, [&inLoop](Ref node) { + if (node[0] == "while") inLoop--; }); } }); } -void optimizeFrounds(Value &ast) { +void optimizeFrounds(Ref ast) { // collapse fround(fround(..)), which can happen due to elimination // also emit f0 instead of fround(0) (except in returns) bool inReturn = false; - std::function fix = [&](Value& node) { - bool ret = node[0] == RETURN; + std::function fix = [&](Ref node) { + bool ret = node[0] == "return"; if (ret) inReturn = true; traverseChildren(node, fix); if (ret) inReturn = false; - if (node[0] == CALL && node[1][0] == NAME && node[1][1] == MATH_FROUND) { - Value& arg = node[2][0]; - if (arg[0] == NUM) { + if (node[0] == "call" && node[1][0] == "name" && node[1][1] == "Math_fround") { + Ref arg = node[2][0]; + if (arg[0] == "num") { if (!inReturn && arg[1] == 0) node = makeName("f0"); - } else if (arg[0] == CALL && arg[1][0] == NAME && arg[1][1] == MATH_FROUND) { + } else if (arg[0] == "call" && arg[1][0] == "name" && arg[1][1] == "Math_fround") { node = arg; } } @@ -567,6 +536,8 @@ int main(int argc, char **argv) { if (comment) *comment = 0; // drop off the comments; TODO: parse extra info // Parse JSON source into a Document + doc.parse(json); +/* doc.Parse(json); assert(!doc.HasParseError()); delete[] json; @@ -584,12 +555,9 @@ int main(int argc, char **argv) { assert(0); } } - +*/ // Emit JSON of modified Document - StringBuffer buffer; - Writer writer(buffer); - doc.Accept(writer); - puts(buffer.GetString()); + doc.stringify(std::cout); return 0; } From e037505db73d83935106800d34df25b6b0554f18 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 3 Nov 2014 17:49:42 -0800 Subject: [PATCH 022/161] misc fixes --- tools/optimizer/minijson.h | 13 +++++++++---- tools/optimizer/optimizer.cpp | 16 +++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 9393e9963b787..161724b70558a 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -18,13 +18,14 @@ struct Value; // Reference to a value. Simple shared_ptr, plus [] operator for convenience - we work on lots of arrays class Ref : public std::shared_ptr { public: + Ref() { + reset(); + } Ref(Value *v) { reset(v); } - Ref& operator[](unsigned x) { - return (*this)[x]; - } + Ref& operator[](unsigned x); // special convenience for comparison to string, which is by value bool operator==(const char *str); @@ -292,7 +293,11 @@ struct ArrayValue : public Value { } }; -// +// Ref methods + +Ref& Ref::operator[](unsigned x) { + return (*get())[x]; +} bool Ref::operator==(const char *str) { return get()->isString() && get()->getString() == str; diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index bb45e34b99585..5a9d43e661ec2 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -15,7 +15,7 @@ // Globals //================== -Value doc; +Ref doc; //================== // Infrastructure @@ -352,7 +352,7 @@ void simplifyIfs(Ref ast) { Ref other = stats[stats->size()-1]; if (other[0] != "if") { // our if block does not end with an if. perhaps if have an else we can flip - if (node->size() >= 4 && node[3][0] == "block") { + if (node->size() >= 4 && node[3][0] == "block") { // XXX node[3] is false, not an array stats = node[3][1]; if (stats->size() == 0) break; other = stats[stats->size()-1]; @@ -535,11 +535,9 @@ int main(int argc, char **argv) { char *comment = strstr(json, "//"); if (comment) *comment = 0; // drop off the comments; TODO: parse extra info - // Parse JSON source into a Document - doc.parse(json); -/* - doc.Parse(json); - assert(!doc.HasParseError()); + // Parse JSON source into the document + doc = new Value(); + doc->parse(json); delete[] json; // Run passes on the Document @@ -555,9 +553,9 @@ int main(int argc, char **argv) { assert(0); } } -*/ + // Emit JSON of modified Document - doc.stringify(std::cout); + doc->stringify(std::cout); return 0; } From ac6a40261e3d301d2a44a2e3453445d8083c660e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 11:11:40 -0800 Subject: [PATCH 023/161] get optimizer building and running without crashes --- tools/optimizer/minijson.h | 9 +++++++-- tools/optimizer/optimizer.cpp | 4 +++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 161724b70558a..c44c5bbe2a1aa 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -27,9 +27,10 @@ class Ref : public std::shared_ptr { Ref& operator[](unsigned x); - // special convenience for comparison to string, which is by value - bool operator==(const char *str); + // special conveniences + bool operator==(const char *str); // comparison to string, which is by value bool operator!=(const char *str); + bool operator!(); // check if null, in effect }; typedef std::vector ArrayStorage; @@ -307,3 +308,7 @@ bool Ref::operator!=(const char *str) { return get()->isString() ? get()->getString() != str : true; } +bool Ref::operator!() { + return get()->isNull(); +} + diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 5a9d43e661ec2..d9efbe6178ec6 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -24,6 +24,7 @@ Ref doc; void dump(const char *str, Ref node) { std::cerr << str << ": "; node->stringify(std::cerr); + std::cerr << std::endl; } int parseInt(const char *str) { @@ -352,7 +353,7 @@ void simplifyIfs(Ref ast) { Ref other = stats[stats->size()-1]; if (other[0] != "if") { // our if block does not end with an if. perhaps if have an else we can flip - if (node->size() >= 4 && node[3][0] == "block") { // XXX node[3] is false, not an array + if (!!node[3] && node[3][0] == "block") { stats = node[3][1]; if (stats->size() == 0) break; other = stats[stats->size()-1]; @@ -556,6 +557,7 @@ int main(int argc, char **argv) { // Emit JSON of modified Document doc->stringify(std::cout); + std::cout << std::endl; return 0; } From 7bc309cd2b336d767e8b6b032f61dc403cb8cb7e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 11:26:35 -0800 Subject: [PATCH 024/161] fix Number parsing --- tools/optimizer/minijson.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index c44c5bbe2a1aa..9d56d0ae89bc7 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -87,7 +87,7 @@ struct Value { } Value& set(double n) { free(); - type = Null; + type = Number; num = n; return *this; } @@ -230,7 +230,7 @@ struct Value { } else { // Number char *after; - num = strtod(curr, &after); + set(strtod(curr, &after)); curr = after; } return curr; From 84f75552aac1b0cfde3ee7d70245abb924fa9e4e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 11:26:43 -0800 Subject: [PATCH 025/161] fix test harness for native optimizer --- tests/test_other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_other.py b/tests/test_other.py index 1efebdc48e181..e0767311e2c6d 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -1945,7 +1945,7 @@ def test_js_optimizer(self): shutil.copyfile(input, input_temp) Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), input_temp, 'emitJSON'], stdin=PIPE, stdout=open(input_temp + '.js', 'w')).communicate() output = Popen([js_optimizer.get_native_optimizer(), input_temp + '.js'] + passes, stdin=PIPE, stdout=open(output_temp, 'w')).communicate()[0] - Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), output_temp, 'emitJSON'], stdin=PIPE, stdout=open(output_temp + '.js', 'w')).communicate() + Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), output_temp, 'receiveJSON'], stdin=PIPE, stdout=open(output_temp + '.js', 'w')).communicate() output = open(output_temp + '.js').read() self.assertIdentical(expected, output.replace('\r\n', '\n').replace('\n\n', '\n')) From 553404f0d0f72033074747b82ce787b0fbc4984a Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 11:39:22 -0800 Subject: [PATCH 026/161] fix up shallow and deep compares --- tools/optimizer/minijson.h | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 9d56d0ae89bc7..cf4801bc5ed15 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -171,11 +171,7 @@ struct Value { case Number: return num == other.num; case Array: - if (arr->size() != other.arr->size()) return false; - for (unsigned i = 0; i < arr->size(); i++) { - if ((*arr)[i] != (*other.arr)[i]) return false; - } - break; + return this == &other; // if you want a deep compare, use deepCompare case Null: break; case Bool: @@ -184,6 +180,18 @@ struct Value { return true; } + bool deepCompare(Ref ref) { + Value& other = *ref; + if (*this == other) return true; // either same pointer, or identical value type (string, number, null or bool) + if (type != other.type) return false; + if (type != Array) return false; // Array is the only one where deep compare differs makes sense, others are shallow and were already tested + if (arr->size() != other.arr->size()) return false; + for (unsigned i = 0; i < arr->size(); i++) { + if ((*arr)[i] != (*other.arr)[i]) return false; + } + return true; + } + char* parse(char* curr) { #define skip() { while (*curr && isspace(*curr)) curr++; } skip(); From 5cedfc6045f67008b79cf0119b18017b1acc1914 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 11:39:37 -0800 Subject: [PATCH 027/161] simplifyIfs fixes --- tools/optimizer/optimizer.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index d9efbe6178ec6..cf9b2102eea22 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -367,11 +367,11 @@ void simplifyIfs(Ref ast) { } else break; } // we can handle elses, but must be fully identical - if (node->size() >= 4 || other->size() >= 4) { - if (!(node->size() >= 4)) break; - if (node[3] != other[3]) { + if (!!node[3] || !!other[3]) { + if (!node[3]) break; + if (node[3]->deepCompare(other[3])) { // the elses are different, but perhaps if we flipped a condition we can do better - if (node[3] == other[2]) { + if (node[3]->deepCompare(other[2])) { // flip other. note that other may not have had an else! add one if so; we will eliminate such things later if (!(other->size() >= 4)) other->push_back(makeBlock()); other[1] = flipCondition(other[1]); @@ -413,12 +413,12 @@ void simplifyIfs(Ref ast) { // we must leave it bool abort = false; - std::unordered_map labelAssigns; + std::unordered_map labelAssigns; traversePre(func, [&labelAssigns, &abort](Ref node) { if (node[0] == "assign" && node[2][0] == "name" && node[2][1] == "label") { if (node[3][0] == "num") { - std::string value = node[3][1]->getString(); + int value = node[3][1]->getNumber(); labelAssigns[value] = labelAssigns[value] + 1; } else { // label is assigned a dynamic value (like from indirectbr), we cannot do anything @@ -428,13 +428,13 @@ void simplifyIfs(Ref ast) { }); if (abort) return; - std::unordered_map labelChecks; + std::unordered_map labelChecks; traversePre(func, [&labelChecks, &abort](Ref node) { if (node[0] == "binary" && node[1] == "==" && node[2][0] == "binary" && node[2][1] == "|" && node[2][2][0] == "name" && node[2][2][1] == "label") { if (node[3][0] == "num") { - std::string value = node[3][1]->getString(); + int value = node[3][1]->getNumber(); labelChecks[value] = labelChecks[value] + 1; } else { // label is checked vs a dynamic value (like from indirectbr), we cannot do anything @@ -461,7 +461,7 @@ void simplifyIfs(Ref ast) { postCond[3][0] == "num") { Ref postValue = postCond[3][1]; Ref preElse = pre[3]; - if (labelAssigns[postValue->getString()] == 1 && labelChecks[postValue->getString()] == 1 && preElse[0] == "block" && preElse->size() >= 2 && preElse[1]->size() == 1) { + if (labelAssigns[postValue->getNumber()] == 1 && labelChecks[postValue->getNumber()] == 1 && preElse[0] == "block" && preElse->size() >= 2 && preElse[1]->size() == 1) { Ref preStat = preElse[1][0]; if (preStat[0] == "stat" && preStat[1][0] == "assign" && preStat[1][1]->isBool(true) && preStat[1][2][0] == "name" && preStat[1][2][1] == "label" && From 9ef8a77e98c63518611375de9e372b4d0c9194bc Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 12:14:39 -0800 Subject: [PATCH 028/161] simplifyIfs porting error fixes --- tools/optimizer/optimizer.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index cf9b2102eea22..59c7e0c7cf2aa 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -21,6 +21,8 @@ Ref doc; // Infrastructure //================== +#define err(str, ...) fprintf(stderr, ...); + void dump(const char *str, Ref node) { std::cerr << str << ": "; node->stringify(std::cerr); @@ -369,11 +371,11 @@ void simplifyIfs(Ref ast) { // we can handle elses, but must be fully identical if (!!node[3] || !!other[3]) { if (!node[3]) break; - if (node[3]->deepCompare(other[3])) { + if (!node[3]->deepCompare(other[3])) { // the elses are different, but perhaps if we flipped a condition we can do better if (node[3]->deepCompare(other[2])) { // flip other. note that other may not have had an else! add one if so; we will eliminate such things later - if (!(other->size() >= 4)) other->push_back(makeBlock()); + if (!other[3]) other[3] = makeBlock(); other[1] = flipCondition(other[1]); Ref temp = other[2]; other[2] = other[3]; @@ -386,7 +388,7 @@ void simplifyIfs(Ref ast) { bool ok = true; for (int i = 0; i < stats->size()-1; i++) { Ref curr = deStat(stats[i]); - if (commable(curr)) ok = false; + if (!commable(curr)) ok = false; } if (!ok) break; for (int i = stats->size()-2; i >= 0; i--) { From 971b76804bc9278f95484b9faf05d924f2b39f92 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 13:29:03 -0800 Subject: [PATCH 029/161] fix deepCompare --- tools/optimizer/minijson.h | 2 +- tools/optimizer/optimizer.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index cf4801bc5ed15..5eb847cef15e9 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -187,7 +187,7 @@ struct Value { if (type != Array) return false; // Array is the only one where deep compare differs makes sense, others are shallow and were already tested if (arr->size() != other.arr->size()) return false; for (unsigned i = 0; i < arr->size(); i++) { - if ((*arr)[i] != (*other.arr)[i]) return false; + if (!(*arr)[i]->deepCompare((*other.arr)[i])) return false; } return true; } diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 59c7e0c7cf2aa..2b13a8434dd49 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -21,7 +21,7 @@ Ref doc; // Infrastructure //================== -#define err(str, ...) fprintf(stderr, ...); +#define err(str) fprintf(stderr, str); void dump(const char *str, Ref node) { std::cerr << str << ": "; @@ -400,7 +400,7 @@ void simplifyIfs(Ref ast) { stats = body[1] = temp; } if (stats->size() != 1) break; - if (node->size() >= 4) simplifiedAnElse = true; + if (!!node[3]) simplifiedAnElse = true; node[1] = make3("conditional", node[1], other[1], makeNum(0)); body = node[2] = other[2]; } From c9ccab8194c5e5fe80b2f6d662a4a284bca26b24 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 14:21:34 -0800 Subject: [PATCH 030/161] final fixes for simplifyIfs --- tools/optimizer/minijson.h | 6 ++++++ tools/optimizer/optimizer.cpp | 25 +++++++++++-------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 5eb847cef15e9..4cd998743af45 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -30,6 +30,7 @@ class Ref : public std::shared_ptr { // special conveniences bool operator==(const char *str); // comparison to string, which is by value bool operator!=(const char *str); + bool operator==(double d) { assert(0); } // prevent Ref == number, which is potentially ambiguous; use ->getNumber() == number bool operator!(); // check if null, in effect }; @@ -290,6 +291,11 @@ struct Value { arr->push_back(r); } + void splice(int x, int num) { + assert(isArray()); + arr->erase(arr->begin() + x, arr->begin() + x + num); + } + // Null operations // Bool operations diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 2b13a8434dd49..fd86d1201190a 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -21,7 +21,7 @@ Ref doc; // Infrastructure //================== -#define err(str) fprintf(stderr, str); +#define err(str) fprintf(stderr, str "\n"); void dump(const char *str, Ref node) { std::cerr << str << ": "; @@ -318,10 +318,6 @@ Ref flipCondition(Ref cond) { return simplifyNotCompsDirect(make2("unary-prefix", "!", cond)); } -void splice(Ref node, int index) { // removes an element from the middle of an array -assert(0);// node.Erase(node.Begin() + index); -} - // Checks bool commable(Ref node) { // TODO: hashing @@ -454,32 +450,32 @@ void simplifyIfs(Ref ast) { for (int i = 0; i < stats->size()-1; i++) { Ref pre = stats[i]; Ref post = stats[i+1]; - if (pre[0] == "if" && pre->size() >= 4 && post[0] == "if" && post->size() < 4) { + if (pre[0] == "if" && !!pre[3] && post[0] == "if" && !post[3]) { Ref postCond = post[1]; if (postCond[0] == "binary" && postCond[1] == "==" && postCond[2][0] == "binary" && postCond[2][1] == "|" && postCond[2][2][0] == "name" && postCond[2][2][1] == "label" && - postCond[2][3][0] == "num" && postCond[2][3][1] == 0 && + postCond[2][3][0] == "num" && postCond[2][3][1]->getNumber() == 0 && postCond[3][0] == "num") { - Ref postValue = postCond[3][1]; + double postValue = postCond[3][1]->getNumber(); Ref preElse = pre[3]; - if (labelAssigns[postValue->getNumber()] == 1 && labelChecks[postValue->getNumber()] == 1 && preElse[0] == "block" && preElse->size() >= 2 && preElse[1]->size() == 1) { + if (labelAssigns[postValue] == 1 && labelChecks[postValue] == 1 && preElse[0] == "block" && preElse->size() >= 2 && preElse[1]->size() == 1) { Ref preStat = preElse[1][0]; if (preStat[0] == "stat" && preStat[1][0] == "assign" && preStat[1][1]->isBool(true) && preStat[1][2][0] == "name" && preStat[1][2][1] == "label" && - preStat[1][3][0] == "num" && preStat[1][3][1] == postValue) { + preStat[1][3][0] == "num" && preStat[1][3][1]->getNumber() == postValue) { // Conditions match, just need to make sure the post clears label if (post[2][0] == "block" && post[2]->size() >= 2 && post[2][1]->size() > 0) { Ref postStat = post[2][1][0]; bool haveClear = postStat[0] == "stat" && postStat[1][0] == "assign" && postStat[1][1]->isBool(true) && postStat[1][2][0] == "name" && postStat[1][2][1] == "label" && - postStat[1][3][0] == "num" && postStat[1][3][1] == 0; + postStat[1][3][0] == "num" && postStat[1][3][1]->getNumber() == 0; if (!inLoop || haveClear) { // Everything lines up, do it pre[3] = post[2]; - if (haveClear) splice(pre[3][1], 0); // remove the label clearing - splice(stats, i+1); // remove the post entirely + if (haveClear) pre[3][1]->splice(0, 1); // remove the label clearing + stats->splice(i+1, 1); // remove the post entirely } } } @@ -491,6 +487,7 @@ void simplifyIfs(Ref ast) { }, [&inLoop](Ref node) { if (node[0] == "while") inLoop--; }); + assert(inLoop == 0); } }); } @@ -507,7 +504,7 @@ void optimizeFrounds(Ref ast) { if (node[0] == "call" && node[1][0] == "name" && node[1][1] == "Math_fround") { Ref arg = node[2][0]; if (arg[0] == "num") { - if (!inReturn && arg[1] == 0) node = makeName("f0"); + if (!inReturn && arg[1]->getNumber() == 0) node = makeName("f0"); } else if (arg[0] == "call" && arg[1][0] == "name" && arg[1][1] == "Math_fround") { node = arg; } From d8b4d111a2524d77e37b99f1b52054d4b3b1cdd3 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 14:44:07 -0800 Subject: [PATCH 031/161] add pretty-printing and an assert --- tools/optimizer/minijson.h | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 4cd998743af45..8e61ae2b153a1 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -245,7 +245,9 @@ struct Value { return curr; } - void stringify(std::ostream &os) { + void stringify(std::ostream &os, bool pretty=false) { + static int indent = 0; + #define indentify() { for (int i = 0; i < indent; i++) os << " "; } switch (type) { case String: os << '"' << *str << '"'; @@ -255,10 +257,23 @@ struct Value { break; case Array: os << '['; + if (pretty) { + os << std::endl; + indent++; + } for (unsigned i = 0; i < arr->size(); i++) { - if (i > 0) os << ", "; - (*arr)[i]->stringify(os); + if (i > 0) { + os << ", "; + if (pretty) os << std::endl; + } + indentify(); + (*arr)[i]->stringify(os, pretty); + } + if (pretty) { + os << std::endl; + indent--; } + indentify(); os << ']'; break; case Null: @@ -283,6 +298,7 @@ struct Value { Ref& operator[](unsigned x) { assert(isArray()); + assert(x < arr->size()); return (*arr)[x]; } From 436500292c1e7568b2e0a316015c0536db552a90 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 14:54:06 -0800 Subject: [PATCH 032/161] make Value[] tolerant, return Null on out of bounds --- tools/optimizer/minijson.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 8e61ae2b153a1..7ad0dab08e274 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -296,9 +296,10 @@ struct Value { return arr->size(); } - Ref& operator[](unsigned x) { + Ref& operator[](unsigned x) { // tolerant, returns Null on out of bounds access. makes it convenient to check e.g. [3] on an if node + static Ref null = new Value(); // TODO: freeze this assert(isArray()); - assert(x < arr->size()); + if (x >= arr->size()) return null; return (*arr)[x]; } From 8ba3dca6a4a3003c14efb0a050d9a9fb7e32afc5 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 14:54:28 -0800 Subject: [PATCH 033/161] get detectType on num --- tools/optimizer/optimizer.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index fd86d1201190a..f8cb7b5a87899 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -23,9 +24,9 @@ Ref doc; #define err(str) fprintf(stderr, str "\n"); -void dump(const char *str, Ref node) { +void dump(const char *str, Ref node, bool pretty=false) { std::cerr << str << ": "; - node->stringify(std::cerr); + node->stringify(std::cerr, pretty); std::cerr << std::endl; } @@ -198,7 +199,7 @@ AsmType detectType(Ref node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { switch (node[0]->getString()[0]) { case 'n': { if (node[0] == "num") { - if (strchr(node[1]->getCString(), '.')) return ASM_DOUBLE; + if (fmod(node[1]->getNumber(), 1) != 0) return ASM_DOUBLE; return ASM_INT; } else if (node[0] == "name") { if (asmInfo) { From 92747bfba725376357935a0c8ec5f909a03656ac Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 15:17:10 -0800 Subject: [PATCH 034/161] fix EMCC_DEBUG=2 with native passes --- emcc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/emcc b/emcc index d8d28bcda23fe..facf7c7737ad9 100755 --- a/emcc +++ b/emcc @@ -1415,14 +1415,18 @@ try: native = True curr.append(p) if len(curr) > 0: - assert not native # FIXME, need to add JSON notes + if native: + curr = ['receiveJSON'] + curr + ['emitJSON'] chunks.append(curr) + if native: + chunks.append(['receiveJSON']) if len(chunks) == 1: run_passes(chunks[0], title, just_split=False, just_concat=False) else: for i in range(len(chunks)): run_passes(chunks[i], 'js_opts_' + str(i), just_split='receiveJSON' in chunks[i], just_concat='emitJSON' in chunks[i]) else: + # DEBUG 2, run each pass seperately for p in passes: js_optimizer_queue = [p] flush_js_optimizer_queue(p) From 128cb505e8a291152a9ff8c97cde9e1d2ad4e422 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 15:17:40 -0800 Subject: [PATCH 035/161] emit full float precision when stringifying --- tools/optimizer/minijson.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 7ad0dab08e274..9106afcea1f6e 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -12,6 +12,7 @@ #include #include #include +#include struct Value; @@ -253,7 +254,7 @@ struct Value { os << '"' << *str << '"'; break; case Number: - os << num; + os << std::setprecision(17) << num; // doubles can have 17 digits of precision break; case Array: os << '['; From b45294cd5aee2bfcce49360e3a1a1701c483afe2 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 15:30:11 -0800 Subject: [PATCH 036/161] some optimizer fixes --- tools/optimizer/optimizer.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index f8cb7b5a87899..e96a92f27a9fc 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -23,6 +23,7 @@ Ref doc; //================== #define err(str) fprintf(stderr, str "\n"); +#define errv(str, ...) fprintf(stderr, str "\n", __VA_ARGS__); void dump(const char *str, Ref node, bool pretty=false) { std::cerr << str << ": "; @@ -290,22 +291,22 @@ Ref simplifyNotCompsDirect(Ref node) { case '<': { if (op == "<") { op->set(">="); break; } if (op == "<=") { op->set(">"); break; } - assert(0); + return node; } case '>': { if (op == ">") { op->set("<="); break; } if (op == ">=") { op->set("<"); break; } - assert(0); + return node; } case '=': { if (op == "==") { op->set("!="); break; } - assert(0); + return node; } case '!': { if (op == "!=") { op->set("=="); break; } - assert(0); + return node; } - default: assert(0); + default: return node; } return make3("binary", op, node[2][2], node[2][3]); } else if (node[2][0] == "unary-prefix" && node[2][1] == "!") { @@ -447,7 +448,7 @@ void simplifyIfs(Ref ast) { traversePrePost(func, [&inLoop, &labelAssigns, &labelChecks](Ref node) { if (node[0] == "while") inLoop++; Ref stats = getStatements(node); - if (!stats->isNull()) { + if (!stats->isNull() && stats->size() > 0) { for (int i = 0; i < stats->size()-1; i++) { Ref pre = stats[i]; Ref post = stats[i+1]; From 1bb13aade5b9de4725eb4e8c2aed1c7a559da738 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 15:51:32 -0800 Subject: [PATCH 037/161] fix optimizeFrounds and fix js_optimizer testing for native passes to only run where possible, adding an frounds-only pass so that is tested --- tests/test_other.py | 4 +++- tools/js_optimizer.py | 2 +- tools/optimizer/optimizer.cpp | 4 ++-- ...-js-optimizer-asm-pre-output-f32-nosimp.js | 23 +++++++++++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 tools/test-js-optimizer-asm-pre-output-f32-nosimp.js diff --git a/tests/test_other.py b/tests/test_other.py index e0767311e2c6d..72aa044347804 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -1910,6 +1910,8 @@ def test_js_optimizer(self): ['asm', 'simplifyExpressions']), (path_from_root('tools', 'test-js-optimizer-asm-pre-f32.js'), open(path_from_root('tools', 'test-js-optimizer-asm-pre-output-f32.js')).read(), ['asm', 'asmPreciseF32', 'simplifyExpressions', 'optimizeFrounds']), + (path_from_root('tools', 'test-js-optimizer-asm-pre-f32.js'), open(path_from_root('tools', 'test-js-optimizer-asm-pre-output-f32-nosimp.js')).read(), + ['asm', 'asmPreciseF32', 'optimizeFrounds']), (path_from_root('tools', 'test-js-optimizer-asm-last.js'), open(path_from_root('tools', 'test-js-optimizer-asm-last-output.js')).read(), ['asm', 'asmLastOpts', 'last']), (path_from_root('tools', 'test-js-optimizer-asm-relocate.js'), open(path_from_root('tools', 'test-js-optimizer-asm-relocate-output.js')).read(), @@ -1936,7 +1938,7 @@ def test_js_optimizer(self): print ' js' output = Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), input] + passes, stdin=PIPE, stdout=PIPE).communicate()[0] self.assertIdentical(expected, output.replace('\r\n', '\n').replace('\n\n', '\n')) - if len(js_optimizer.NATIVE_PASSES.intersection(passes)) > 0: + if len(js_optimizer.NATIVE_PASSES.intersection(passes)) == len(passes): # test calling native print ' native' self.clear() diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 2a058cfd6b02b..b0b9fdcb57ca7 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -9,7 +9,7 @@ def path_from_root(*pathelems): return os.path.join(__rootpath__, *pathelems) -NATIVE_PASSES = set(['simplifyIfs', 'optimizeFrounds']) +NATIVE_PASSES = set(['asm', 'asmPreciseF32', 'simplifyIfs', 'optimizeFrounds']) JS_OPTIMIZER = path_from_root('tools', 'js-optimizer.js') diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index e96a92f27a9fc..f015af9802d36 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -506,9 +506,9 @@ void optimizeFrounds(Ref ast) { if (node[0] == "call" && node[1][0] == "name" && node[1][1] == "Math_fround") { Ref arg = node[2][0]; if (arg[0] == "num") { - if (!inReturn && arg[1]->getNumber() == 0) node = makeName("f0"); + if (!inReturn && arg[1]->getNumber() == 0) *node = *makeName("f0"); } else if (arg[0] == "call" && arg[1][0] == "name" && arg[1][1] == "Math_fround") { - node = arg; + *node = *arg; } } }; diff --git a/tools/test-js-optimizer-asm-pre-output-f32-nosimp.js b/tools/test-js-optimizer-asm-pre-output-f32-nosimp.js new file mode 100644 index 0000000000000..3bbb4689f8547 --- /dev/null +++ b/tools/test-js-optimizer-asm-pre-output-f32-nosimp.js @@ -0,0 +1,23 @@ +function badf() { + var $9 = f0; + $9 = (HEAP32[tempDoublePtr >> 2] = $8, Math_fround(HEAPF32[tempDoublePtr >> 2])); + HEAPF32[$gep23_asptr >> 2] = $9; +} +function badf2() { + var $9 = 0; + $9 = (HEAPF32[tempDoublePtr >> 2] = $8, HEAP32[tempDoublePtr >> 2] | 0); + HEAP32[$gep23_asptr >> 2] = $9; +} +function dupe() { + x = Math_fround(x); + x = Math_fround(x); + x = Math_fround(x); + x = Math_fround(x); +} +function zeros(x) { + x = Math_fround(x); + var y = f0; + print(Math_fround(y) + f0); + return Math_fround(0); +} + From f036f6813e8f8a5c8bad0e3502e3ce05e7760697 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 15:58:45 -0800 Subject: [PATCH 038/161] fix js_optimizer.py decision when to use the native optimizer --- tools/js_optimizer.py | 2 +- tools/optimizer/minijson.h | 4 ++++ tools/optimizer/optimizer.cpp | 4 ---- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index b0b9fdcb57ca7..14227d5e4b8ec 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -290,7 +290,7 @@ def write_chunk(chunk, i): filenames = [] if len(filenames) > 0: - if len(NATIVE_PASSES.intersection(passes)) == 0: + if len(NATIVE_PASSES.intersection(passes)) != len(passes): commands = map(lambda filename: js_engine + [JS_OPTIMIZER, filename, 'noPrintMetadata'] + (['--debug'] if source_map else []) + passes, filenames) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 9106afcea1f6e..5fbb4ff8ce31d 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -8,12 +8,16 @@ #include #include #include +#include #include #include #include #include +#define err(str) fprintf(stderr, str "\n"); +#define errv(str, ...) fprintf(stderr, str "\n", __VA_ARGS__); + struct Value; // Reference to a value. Simple shared_ptr, plus [] operator for convenience - we work on lots of arrays diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index f015af9802d36..28787c99c747c 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -22,9 +21,6 @@ Ref doc; // Infrastructure //================== -#define err(str) fprintf(stderr, str "\n"); -#define errv(str, ...) fprintf(stderr, str "\n", __VA_ARGS__); - void dump(const char *str, Ref node, bool pretty=false) { std::cerr << str << ": "; node->stringify(std::cerr, pretty); From d26fcc824e97ff52ac246a6fdb7daec551f833af Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 16:59:46 -0800 Subject: [PATCH 039/161] avoid isspace --- tools/optimizer/minijson.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 5fbb4ff8ce31d..1b3d9c370dd7b 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -4,7 +4,6 @@ * Uses shared_ptr for simplicity, basically everywhere. TODO: measure impact */ -#include #include #include #include @@ -199,7 +198,8 @@ struct Value { } char* parse(char* curr) { - #define skip() { while (*curr && isspace(*curr)) curr++; } + #define is_json_space(x) (x == 32 || x == 9 || x == 10 || x == 13) /* space, tab, linefeed/newline, or return */ + #define skip() { while (*curr && is_json_space(*curr)) curr++; } skip(); if (*curr == '"') { // String From db463c7fed070564f45a48758fb1fccaaf5499dc Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 4 Nov 2014 17:11:11 -0800 Subject: [PATCH 040/161] unify decisions on whether to run natively, and take into account source maps, which native cannot do --- emcc | 2 +- tests/test_other.py | 2 +- tools/js_optimizer.py | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/emcc b/emcc index facf7c7737ad9..38dd2dea59d44 100755 --- a/emcc +++ b/emcc @@ -1400,7 +1400,7 @@ try: curr = [] native = False for p in passes: - if p not in shared.js_optimizer.NATIVE_PASSES: + if not shared.js_optimizer.use_native([p], source_map=debug_level >= 4): if native: chunks.append(['receiveJSON'] + curr + ['emitJSON']) curr = [] diff --git a/tests/test_other.py b/tests/test_other.py index 72aa044347804..eb5d79871a342 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -1938,7 +1938,7 @@ def test_js_optimizer(self): print ' js' output = Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), input] + passes, stdin=PIPE, stdout=PIPE).communicate()[0] self.assertIdentical(expected, output.replace('\r\n', '\n').replace('\n\n', '\n')) - if len(js_optimizer.NATIVE_PASSES.intersection(passes)) == len(passes): + if js_optimizer.use_native(passes): # test calling native print ' native' self.clear() diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 14227d5e4b8ec..35e654e2ef771 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -9,7 +9,7 @@ def path_from_root(*pathelems): return os.path.join(__rootpath__, *pathelems) -NATIVE_PASSES = set(['asm', 'asmPreciseF32', 'simplifyIfs', 'optimizeFrounds']) +NATIVE_PASSES = set(['asm', 'asmPreciseF32', 'receiveJSON', 'emitJSON', 'simplifyIfs', 'optimizeFrounds']) JS_OPTIMIZER = path_from_root('tools', 'js-optimizer.js') @@ -34,6 +34,9 @@ def create_optimizer(): return output return shared.Cache.get('optimizer.exe', create_optimizer, extension='exe') +def use_native(passes, source_map=False): + return (not source_map) and len(NATIVE_PASSES.intersection(passes)) == len(passes) + class Minifier: ''' asm.js minification support. We calculate minification of @@ -290,7 +293,7 @@ def write_chunk(chunk, i): filenames = [] if len(filenames) > 0: - if len(NATIVE_PASSES.intersection(passes)) != len(passes): + if not use_native(passes, source_map): commands = map(lambda filename: js_engine + [JS_OPTIMIZER, filename, 'noPrintMetadata'] + (['--debug'] if source_map else []) + passes, filenames) From 4303279317be8f5de8fed9f78ace749bf89d68f8 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 5 Nov 2014 17:03:00 -0800 Subject: [PATCH 041/161] initial work to convert simplifyExpressions to new optimizer --- tools/js_optimizer.py | 4 +- tools/optimizer/minijson.h | 65 ++- tools/optimizer/optimizer.cpp | 945 +++++++++++++++++++++++++++++++--- 3 files changed, 916 insertions(+), 98 deletions(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 35e654e2ef771..b97c02b4e39d9 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -9,7 +9,7 @@ def path_from_root(*pathelems): return os.path.join(__rootpath__, *pathelems) -NATIVE_PASSES = set(['asm', 'asmPreciseF32', 'receiveJSON', 'emitJSON', 'simplifyIfs', 'optimizeFrounds']) +NATIVE_PASSES = set(['asm', 'asmPreciseF32', 'receiveJSON', 'emitJSON', 'simplifyExpressions', 'simplifyIfs', 'optimizeFrounds']) JS_OPTIMIZER = path_from_root('tools', 'js-optimizer.js') @@ -29,7 +29,7 @@ def create_optimizer(): shared.logging.debug('building native optimizer') output = shared.Cache.get_path('optimizer.exe') shared.try_delete(output) - subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-I' + shared.path_from_root('tools', 'optimizer', 'rapidjson'), '-O2', '-std=c++11', '-o', output]).communicate() + subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-I' + shared.path_from_root('tools', 'optimizer', 'rapidjson'), '-O2', '-std=c++11', '-o', output, '-g']).communicate() assert os.path.exists(output) return output return shared.Cache.get('optimizer.exe', create_optimizer, extension='exe') diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 1b3d9c370dd7b..fb903147a4e8c 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -13,6 +13,7 @@ #include #include #include +#include #define err(str) fprintf(stderr, str "\n"); #define errv(str, ...) fprintf(stderr, str "\n", __VA_ARGS__); @@ -62,10 +63,13 @@ struct Value { // constructors all copy their input Value() : type(Null), num(0) {} explicit Value(const char *s) : type(Null) { - set(s); + setString(s); + } + explicit Value(const std::string& s) : type(Null) { + setString(s.c_str()); } explicit Value(double n) : type(Null) { - set(n); + setNumber(n); } explicit Value(ArrayStorage &a) : type(Null) { setArray(); @@ -84,19 +88,22 @@ struct Value { num = 0; } - Value& set(const char *s) { + Value& setString(const char *s) { free(); type = String; str = new std::string(s); return *this; } - Value& set(double n) { + Value& setString(const std::string& s) { + return setString(s.c_str()); + } + Value& setNumber(double n) { free(); type = Number; num = n; return *this; } - Value& set(ArrayStorage &a) { + Value& setArray(ArrayStorage &a) { free(); type = Array; arr = new ArrayStorage(); @@ -141,6 +148,10 @@ struct Value { assert(isNumber()); return num; } + ArrayStorage& getArray() { + assert(isArray()); + return *arr; + } bool& getBool() { assert(isBool()); return boo; @@ -150,13 +161,13 @@ struct Value { free(); switch (other.type) { case String: - set(other.str->c_str()); + setString(other.str->c_str()); break; case Number: - set(other.num); + setNumber(other.num); break; case Array: - set(*other.arr); + setArray(*other.arr); break; case Null: setNull(); @@ -207,7 +218,7 @@ struct Value { char *close = strchr(curr, '"'); assert(close); *close = 0; - set(curr); + setString(curr); *close = '"'; curr = close+1; } else if (*curr == '[') { @@ -244,7 +255,7 @@ struct Value { } else { // Number char *after; - set(strtod(curr, &after)); + setNumber(strtod(curr, &after)); curr = after; } return curr; @@ -301,6 +312,17 @@ struct Value { return arr->size(); } + void setSize(unsigned size) { + assert(isArray()); + unsigned old = arr->size(); + if (old != size) arr->resize(size); + if (old < size) { + for (unsigned i = old; i < size; i++) { + (*arr)[i] = new Value(); + } + } + } + Ref& operator[](unsigned x) { // tolerant, returns Null on out of bounds access. makes it convenient to check e.g. [3] on an if node static Ref null = new Value(); // TODO: freeze this assert(isArray()); @@ -308,9 +330,10 @@ struct Value { return (*arr)[x]; } - void push_back(Ref r) { + Value& push_back(Ref r) { assert(isArray()); arr->push_back(r); + return *this; } void splice(int x, int num) { @@ -318,6 +341,26 @@ struct Value { arr->erase(arr->begin() + x, arr->begin() + x + num); } + void insert(int x, int num) { + arr->insert(arr->begin() + x, num, Ref()); + } + + int indexOf(Ref other) { + assert(isArray()); + for (unsigned i = 0; i < arr->size(); i++) { + if (other == (*arr)[i]) return i; + } + return -1; + } + + /* + void forEach(std::function func) { + for (unsigned i = 0; i < arr->size(); i++) { + func((*arr)[i]); + } + } + */ + // Null operations // Bool operations diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 28787c99c747c..c1903e73a279c 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -2,11 +2,12 @@ #include #include #include +#include #include #include +#include #include -#include #include #include "minijson.h" @@ -36,6 +37,18 @@ int parseInt(const char *str) { return ret; } +std::string getIntStr(int x) { + switch (x) { + case 1: return "1"; + case 8: return "8"; + case 16: return "16"; + case 32: return "32"; + case 64: return "64"; + } + assert(0); + return ":("; +} + // Traversals // Traverses the children of a node. @@ -60,7 +73,7 @@ void traverseChildren(Ref node, std::function visit) { // Traverse, calling visit before the children void traversePre(Ref node, std::function visit) { visit(node); - traverseChildren(node, [&visit](Ref node) { + traverseChildren(node, [&visit](Ref node) { traversePre(node, visit); }); } @@ -68,7 +81,7 @@ void traversePre(Ref node, std::function visit) { // Traverse, calling visitPre before the children and visitPost after void traversePrePost(Ref node, std::function visitPre, std::function visitPost) { visitPre(node); - traverseChildren(node, [&visitPre, &visitPost](Ref node) { + traverseChildren(node, [&visitPre, &visitPost](Ref node) { traversePrePost(node, visitPre, visitPost); }); visitPost(node); @@ -87,12 +100,12 @@ void traverseFunctions(Ref ast, std::function visit) { } } -Ref deStat(Ref node) { +Ref deStat(Ref node) { if (node[0]->getString() =="stat") return node[1]; return node; } -Ref getStatements(Ref node) { +Ref getStatements(Ref node) { if (node[0] == "defun") { return node[3]; } else if (node[0] == "block") { @@ -102,54 +115,6 @@ Ref getStatements(Ref node) { } } -// Constructions TODO: share common constructions, and assert they remain frozen - -Ref makeName(const char *str) { - Ref ret(new ArrayValue()); - ret->push_back(new Value("name")); - ret->push_back(new Value(str)); - return ret; -} - -Ref makeNum(double x) { - Ref ret(new ArrayValue()); - ret->push_back(new Value("num")); - ret->push_back(new Value(x)); - return ret; -} - -Ref makeBlock() { - Ref ret(new ArrayValue()); - ret->push_back(new Value("block")); - ret->push_back(new ArrayValue()); - return ret; -} - -Ref make2(const char* type, const char *a, Ref b) { - Ref ret(new ArrayValue()); - ret->push_back(new Value(type)); - ret->push_back(new Value(a)); - ret->push_back(b); - return ret; -} - -Ref make2(const char* type, Ref a, Ref b) { - Ref ret(new ArrayValue()); - ret->push_back(new Value(type)); - ret->push_back(a); - ret->push_back(b); - return ret; -} - -Ref make3(const char *type, Ref a, Ref b, Ref c) { - Ref ret(new ArrayValue()); - ret->push_back(new Value(type)); - ret->push_back(a); - ret->push_back(b); - ret->push_back(c); - return ret; -} - // Types enum AsmType { @@ -161,9 +126,12 @@ enum AsmType { ASM_NONE = 5 }; -struct AsmInfo { +struct AsmData { std::map params; // name => index std::map types; // name => type + AsmType ret; + + AsmData() : ret(ASM_NONE) {} AsmType getType(std::string name) { auto ret = types.find(name); @@ -192,15 +160,25 @@ HeapInfo parseHeap(const char *name) { return ret; } -AsmType detectType(Ref node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { +bool isInteger(double x) { + return fmod(x, 1) == 0; +} + +bool isInteger32(double x) { + return isInteger(x) && (x == (int32_t)x || x == (uint32_t)x); +} + +std::string ASM_FLOAT_ZERO; + +AsmType detectType(Ref node, AsmData *asmData=nullptr, bool inVarDef=false) { switch (node[0]->getString()[0]) { case 'n': { if (node[0] == "num") { - if (fmod(node[1]->getNumber(), 1) != 0) return ASM_DOUBLE; + if (!isInteger(node[1]->getNumber())) return ASM_DOUBLE; return ASM_INT; } else if (node[0] == "name") { - if (asmInfo) { - AsmType ret = asmInfo->getType(node[1]->getString()); + if (asmData) { + AsmType ret = asmData->getType(node[1]->getString()); if (ret != ASM_NONE) return ret; } if (!inVarDef) { @@ -209,12 +187,9 @@ AsmType detectType(Ref node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { return ASM_NONE; } // We are in a variable definition, where Math_fround(0) optimized into a global constant becomes f0 = Math_fround(0) - assert(0); - /* - if (!ASM_FLOAT_ZERO) ASM_FLOAT_ZERO = node[1]; - else assert(ASM_FLOAT_ZERO == node[1]); + if (ASM_FLOAT_ZERO.size() == 0) ASM_FLOAT_ZERO = node[1]->getString(); + else assert(ASM_FLOAT_ZERO == node[1]->getString()); return ASM_FLOAT; - */ } break; } @@ -222,7 +197,7 @@ AsmType detectType(Ref node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { if (node[0] == "unary-prefix") { switch (node[1]->getString()[0]) { case '+': return ASM_DOUBLE; - case '-': return detectType(node[2], asmInfo, inVarDef); + case '-': return detectType(node[2], asmData, inVarDef); case '!': case '~': return ASM_INT; } break; @@ -239,7 +214,7 @@ AsmType detectType(Ref node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { } return ASM_NONE; } else if (node[0] == "conditional") { - return detectType(node[2], asmInfo, inVarDef); + return detectType(node[2], asmData, inVarDef); } break; } @@ -247,7 +222,7 @@ AsmType detectType(Ref node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { if (node[0] == "binary") { switch (node[1]->getString()[0]) { case '+': case '-': - case '*': case '/': case '%': return detectType(node[2], asmInfo, inVarDef); + case '*': case '/': case '%': return detectType(node[2], asmData, inVarDef); case '|': case '&': case '^': case '<': case '>': // handles <<, >>, >>=, <=, >= case '=': case '!': { // handles ==, != return ASM_INT; @@ -258,7 +233,7 @@ AsmType detectType(Ref node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { } case 's': { if (node[0] == "seq") { - return detectType(node[2], asmInfo, inVarDef); + return detectType(node[2], asmData, inVarDef); } else if (node[0] == "sub") { assert(node[1][0] == "name"); HeapInfo info = parseHeap(node[1][1]->getCString()); @@ -272,7 +247,312 @@ AsmType detectType(Ref node, AsmInfo *asmInfo=nullptr, bool inVarDef=false) { assert(0); } +// Constructions TODO: share common constructions, and assert they remain frozen + +Ref makeEmpty() { + Ref ret(new ArrayValue()); + ret->push_back(new Value("toplevel")); + ret->push_back(new ArrayValue()); + return ret; +} + +Ref makePair(Ref x, Ref y) { + Ref ret = new ArrayValue(); + ret->push_back(x); + ret->push_back(y); + return ret; +}; + +Ref makeNum(double x) { + Ref ret(new ArrayValue()); + ret->push_back(new Value("num")); + ret->push_back(new Value(x)); + return ret; +} + +Ref makeName(const char *str) { + Ref ret(new ArrayValue()); + ret->push_back(new Value("name")); + ret->push_back(new Value(str)); + return ret; +} + +Ref makeName(const std::string& str) { + return makeName(str.c_str()); +} + +Ref makeBlock() { + Ref ret(new ArrayValue()); + ret->push_back(new Value("block")); + ret->push_back(new ArrayValue()); + return ret; +} + +Ref make1(const char* type, Ref a) { + Ref ret(new ArrayValue()); + ret->push_back(new Value(type)); + ret->push_back(a); + return ret; +} + +Ref make2(const char* type, const char *a, Ref b) { + Ref ret(new ArrayValue()); + ret->push_back(new Value(type)); + ret->push_back(new Value(a)); + ret->push_back(b); + return ret; +} + +Ref make2(const char* type, Ref a, Ref b) { + Ref ret(new ArrayValue()); + ret->push_back(new Value(type)); + ret->push_back(a); + ret->push_back(b); + return ret; +} + +Ref make3(const char *type, const char *a, Ref b, Ref c) { + Ref ret(new ArrayValue()); + ret->push_back(new Value(type)); + ret->push_back(new Value(a)); + ret->push_back(b); + ret->push_back(c); + return ret; +} + +Ref make3(const char *type, Ref a, Ref b, Ref c) { + Ref ret(new ArrayValue()); + ret->push_back(new Value(type)); + ret->push_back(a); + ret->push_back(b); + ret->push_back(c); + return ret; +} + +Ref makeAsmVarDef(const std::string& v_, AsmType type) { + Ref v = new Value(v_); + Ref val; + switch (type) { + case ASM_INT: val = makeNum(0); break; + case ASM_DOUBLE: val = make2("unary-prefix", "+", makeNum(0)); break; + case ASM_FLOAT: { + if (ASM_FLOAT_ZERO.size() > 0) { + val = makeName(ASM_FLOAT_ZERO.c_str()); + } else { + return make2("call", makeName("Math_fround"), &(new ArrayValue())->push_back(makeNum(0))); + } + break; + } + case ASM_FLOAT32X4: { + val = make2("call", makeName("SIMD_float32x4"), &(new ArrayValue())->push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0))); + break; + } + case ASM_INT32X4: { + val = make2("call", makeName("SIMD_int32x4"), &(new ArrayValue())->push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0))); + break; + } + default: assert(0); + } + return makePair(v, val); +} + +Ref makeAsmCoercion(Ref node, AsmType type) { + switch (type) { + case ASM_INT: return make3("binary", "|", node, makeNum(0)); + case ASM_DOUBLE: return make2("unary-prefix", "+", node); + case ASM_FLOAT: return make2("call", makeName("Math_fround"), &(new ArrayValue())->push_back(node)); + case ASM_FLOAT32X4: return make2("call", makeName("SIMD_float32x4"), &(new ArrayValue())->push_back(node)); + case ASM_INT32X4: return make2("call", makeName("SIMD_int32x4"), &(new ArrayValue())->push_back(node)); + case ASM_NONE: + default: return node; // non-validating code, emit nothing XXX this is dangerous, we should only allow this when we know we are not validating + } +} + +// Checks + +bool isEmpty(Ref node) { + return node->size() == 2 && node[0] == "toplevel" && node[1]->size() == 0; +} + +bool commable(Ref node) { // TODO: hashing + std::string type = node[0]->getString(); + if (type == "assign" || type == "binary" || type == "unary-prefix" || type == "unary-postfix" || type == "name" || type == "num" || type == "call" || type == "seq" || type == "conditional" || type == "sub") return true; + return false; +} + +bool isMathFunc(const char *name) { + static const char *Math_ = "Math_"; + static unsigned size = strlen(Math_); + return strncmp(name, Math_, size) == 0; +} + +bool isMathFunc(Ref value) { + return value->isString() && isMathFunc(value->getString().c_str()); +} + +bool callHasSideEffects(Ref node) { // checks if the call itself (not the args) has side effects (or is not statically known) + return !(node[1][0] == "name" && isMathFunc(node[1][1])); +} + +bool hasSideEffects(Ref node) { // this is 99% incomplete! + std::string& type = node[0]->getString(); + switch (type[0]) { + case 'n': + if (type == "num" || type == "name") return false; + break; + case 's': + if (type == "string") return false; + if (type == "sub") return hasSideEffects(node[1]) || hasSideEffects(node[2]); + break; + case 'u': + if (type == "unary-prefix") return hasSideEffects(node[2]); + break; + case 'b': + if (type == "binary") return hasSideEffects(node[2]) || hasSideEffects(node[3]); + break; + case 'c': + if (type == "call") { + if (callHasSideEffects(node)) return true; + // This is a statically known call, with no side effects. only args can side effect us + for (auto arg : node[2]->getArray()) { + if (hasSideEffects(arg)) return true; + } + return false; + } else if (type == "conditional") return hasSideEffects(node[1]) || hasSideEffects(node[2]) || hasSideEffects(node[3]); + break; + } + return true; +} + // Transforms +/* +struct AsmData { + std::map params; // name => index + std::map types; // name => type + type ret + + AsmType getType(std::string name) { + auto ret = types.find(name); + if (ret != types.end()) return ret->second; + return ASM_NONE; + } +}; +*/ +AsmData normalizeAsm(Ref func) { + AsmData data; + // process initial params + Ref stats = func[3]; + int i = 0; + while (i < stats->size()) { + Ref node = stats[i]; + if (node[0] != "stat" || node[1][0] != "assign" || node[1][2][0] != "name") break; + node = node[1]; + Ref name = node[2][1]; + if (!!func[2] && func[2]->indexOf(name) < 0) break; // not an assign into a parameter, but a global + std::string& str = name->getString(); + if (data.params.count(str) > 0) break; // already done that param, must be starting function body + data.params[str] = func[2]->indexOf(name); + data.types[str] = detectType(node[3]); + stats[i] = makeEmpty(); + i++; + } + // process initial variable definitions + while (i < stats->size()) { + Ref node = stats[i]; + if (node[0] != "var") break; + for (int j = 0; j < node[1]->size(); j++) { + Ref v = node[1][j]; + std::string name = v[0]->getString(); + Ref value = v[1]; + if (!data.types.count(name)) { + data.types[name] = detectType(value, nullptr, true); + v->setSize(1); // make an un-assigning var + } else { + assert(j == 0); // cannot break in the middle + goto outside; + } + } + i++; + } + outside: + // look for other var definitions and collect them + while (i < stats->size()) { + traversePre(stats[i], [](Ref node) { + Ref type = node[0]; + if (type == "var") { + assert(0); //, 'should be no vars to fix! ' + func[1] + ' : ' + JSON.stringify(node)); + } + }); + i++; + } + // look for final "return" statement to get return type. + Ref retStmt = stats[stats->size() - 1]; + if (!!retStmt && retStmt[0] == "return" && !!retStmt[1]) { + data.ret = detectType(retStmt[1]); + } + return data; +} + +void denormalizeAsm(Ref func, AsmData& data) { + Ref stats = func[3]; + // Remove var definitions, if any + for (int i = 0; i < stats->size(); i++) { + if (stats[i][0] == "var") { + stats[i] = makeEmpty(); + } else { + if (!isEmpty(stats[i])) break; + } + } + // calculate variable definitions + Ref varDefs = new ArrayValue(); + for (auto v : data.types) { + varDefs->push_back(makeAsmVarDef(v.first, v.second)); + } + // each param needs a line; reuse emptyNodes as much as we can + int numParams = data.params.size(); + int emptyNodes = 0; + while (emptyNodes < stats->size()) { + if (!isEmpty(stats[emptyNodes])) break; + emptyNodes++; + } + int neededEmptyNodes = numParams + (varDefs->size() ? 1 : 0); // params plus one big var if there are vars + if (neededEmptyNodes > emptyNodes) { + stats->insert(0, neededEmptyNodes - emptyNodes); + } else if (neededEmptyNodes < emptyNodes) { + stats->splice(0, emptyNodes - neededEmptyNodes); + } + // add param coercions + int next = 0; + for (auto param : func[2]->getArray()) { + std::string& str = param->getString(); + stats[next++] = make1("stat", make3("assign", &(new Value())->setBool(true), makeName(str), makeAsmCoercion(makeName(str), data.types[str]))); + } + if (varDefs->size()) { + stats[next] = make1("var", varDefs); + } + /* + if (data.inlines.length > 0) { + var i = 0; + traverse(func, function(node, type) { + if (type == "call" && node[1][0] == "name" && node[1][1] == 'inlinejs') { + node[1] = data.inlines[i++]; // swap back in the body + } + }); + } + */ + // ensure that there's a final "return" statement if needed. + if (data.ret != ASM_NONE) { + Ref retStmt = stats[stats->size() - 1]; + if (!retStmt || retStmt[0]->getString() != "return") { + Ref retVal = makeNum(0); + if (data.ret != ASM_INT) { + retVal = makeAsmCoercion(retVal, data.ret); + } + stats->push_back(make1("return", retVal)); + } + } + //printErr('denormalized \n\n' + astToSrc(func) + '\n\n'); +} // We often have branchings that are simplified so one end vanishes, and // we then get @@ -285,21 +565,21 @@ Ref simplifyNotCompsDirect(Ref node) { Ref op = node[2][1]; switch(op->getCString()[0]) { case '<': { - if (op == "<") { op->set(">="); break; } - if (op == "<=") { op->set(">"); break; } + if (op == "<") { op->setString(">="); break; } + if (op == "<=") { op->setString(">"); break; } return node; } case '>': { - if (op == ">") { op->set("<="); break; } - if (op == ">=") { op->set("<"); break; } + if (op == ">") { op->setString("<="); break; } + if (op == ">=") { op->setString("<"); break; } return node; } case '=': { - if (op == "==") { op->set("!="); break; } + if (op == "==") { op->setString("!="); break; } return node; } case '!': { - if (op == "!=") { op->set("=="); break; } + if (op == "!=") { op->setString("=="); break; } return node; } default: return node; @@ -316,12 +596,27 @@ Ref flipCondition(Ref cond) { return simplifyNotCompsDirect(make2("unary-prefix", "!", cond)); } -// Checks +void safeCopy(Ref target, Ref source) { // safely copy source onto target, even if source is a subnode of target + Ref temp = source; // hold on to source + *target = *temp; +} -bool commable(Ref node) { // TODO: hashing - std::string type = node[0]->getString(); - if (type == "assign" || type == "binary" || type == "unary-prefix" || type == "unary-postfix" || type == "name" || type == "num" || type == "call" || type == "seq" || type == "conditional" || type == "sub") return true; - return false; +// Calculations + +int measureCost(Ref ast) { + int size = 0; + traversePre(ast, [&size](Ref node) { + Ref type = node[0]; + if (type == "num" || type == "unary-prefix") size--; + else if (type == "binary") { + if (node[3][0] == "num" && node[3][1]->getNumber() == 0) size--; + else if (node[1] == "/" || node[1] == "%") size += 2; + } + else if (type == "call" && !callHasSideEffects(node)) size -= 2; + else if (type == "sub") size++; + size++; + }); + return size; } //================== @@ -334,11 +629,490 @@ bool preciseF32 = false; // Optimization passes //===================== +class StringSet : public std::unordered_set { +public: + StringSet(const char *init) { // comma-delimited list + int size = strlen(init); + char *curr = (char*)malloc(size+1); // leaked! + strcpy(curr, init); + while (1) { + char *end = strchr(curr, ' '); + if (end) *end = 0; + insert(curr); + if (!end) break; + curr = end + 1; + } + } + + bool has(const char *str) { + return count(str) > 0; + } + bool has(Ref node) { + return has(node->getString().c_str()); + } +}; + +StringSet USEFUL_BINARY_OPS("<< >> | & ^"), + COMPARE_OPS("< <= > >= == == != !=="), + BITWISE("| & ^"), + SAFE_BINARY_OPS("+ -"), // division is unsafe as it creates non-ints in JS; mod is unsafe as signs matter so we can't remove |0's; mul does not nest with +,- in asm + COERCION_REQUIRING_OPS("sub unary-prefix"), // ops that in asm must be coerced right away + COERCION_REQUIRING_BINARIES("* / %"); // binary ops that in asm must be coerced + +bool isFunctionTable(const char *name) { + static const char *functionTable = "FUNCTION_TABLE"; + static unsigned size = strlen(functionTable); + return strncmp(name, functionTable, size) == 0; +} + +bool isFunctionTable(Ref value) { + return value->isString() && isFunctionTable(value->getString().c_str()); +} + +void simplifyExpressions(Ref ast) { + // Simplify common expressions used to perform integer conversion operations + // in cases where no conversion is needed. + auto simplifyIntegerConversions = [](Ref ast) { + traversePre(ast, [](Ref node) { + Ref type = node[0]; + if (type == "binary" && node[1] == ">>" && node[3][0] == "num" && + node[2][0] == "binary" && node[2][1] == "<<" && node[2][3][0] == "num" && node[3][1]->getNumber() == node[2][3][1]->getNumber()) { + // Transform (x&A)<>B to X&A. + Ref innerNode = node[2][2]; + double shifts = node[3][1]->getNumber(); + if (innerNode[0] == "binary" && innerNode[1] == "&" && innerNode[3][0] == "num") { + double mask = innerNode[3][1]->getNumber(); + if (isInteger32(mask) && isInteger32(shifts) && ((int(mask) << int(shifts)) >> int(shifts)) == int(mask)) { + safeCopy(node, innerNode); + return; + } + } + } else if (type == "binary" && BITWISE.has(node[1])) { + for (int i = 2; i <= 3; i++) { + Ref subNode = node[i]; + if (subNode[0] == "binary" && subNode[1] == "&" && subNode[3][0] == "num" && subNode[3][1]->getNumber() == 1) { + // Rewrite (X < Y) & 1 to X < Y , when it is going into a bitwise operator. We could + // remove even more (just replace &1 with |0, then subsequent passes could remove the |0) + // but v8 issue #2513 means the code would then run very slowly in chrome. + Ref input = subNode[2]; + if (input[0] == "binary" && COMPARE_OPS.has(input[1])) { + safeCopy(node[i], input); + } + } + } + } + }); + }; + + // When there is a bunch of math like (((8+5)|0)+12)|0, only the external |0 is needed, one correction is enough. + // At each node, ((X|0)+Y)|0 can be transformed into (X+Y): The inner corrections are not needed + // TODO: Is the same is true for 0xff, 0xffff? + // Likewise, if we have |0 inside a block that will be >>'d, then the |0 is unnecessary because some + // 'useful' mathops already |0 anyhow. + + auto simplifyOps = [](Ref ast) { + auto removeMultipleOrZero = [&ast] { + bool rerun = true; + while (rerun) { + rerun = false; + std::vector stack; + std::function process = [&stack, &rerun, &process, &ast](Ref node) { + Ref type = node[0]; + if (type == "binary" && node[1] == "|") { + if (node[2][0] == "num" && node[3][0] == "num") { + node[2][1]->setNumber(int(node[2][1]->getNumber()) | int(node[3][1]->getNumber())); + stack.push_back(0); + safeCopy(node, node[2]); + return; + } + bool go = false; + if (node[2][0] == "num" && node[2][1]->getNumber() == 0) { + // canonicalize order + Ref temp = node[3]; + node[3] = node[2]; + node[2] = temp; + go = true; + } else if (node[3][0] == "num" && node[3][1]->getNumber() == 0) { + go = true; + } + if (!go) { + stack.push_back(1); + return; + } + // We might be able to remove this correction + for (int i = stack.size()-1; i >= 0; i--) { + if (stack[i] >= 1) { + if (stack[stack.size()-1] < 2 && node[2][0] == "call") break; // we can only remove multiple |0s on these + if (stack[stack.size()-1] < 1 && (COERCION_REQUIRING_OPS.has(node[2][0]) || + (node[2][0] == "binary" && COERCION_REQUIRING_BINARIES.has(node[2][1])))) break; // we can remove |0 or >>2 + // we will replace ourselves with the non-zero side. Recursively process that node. + Ref result = node[2][0] == "num" && node[2][1]->getNumber() == 0 ? node[3] : node[2], other; + // replace node in-place + safeCopy(node, result); + rerun = true; + process(result); + return; + } else if (stack[i] == -1) { + break; // Too bad, we can't + } + } + stack.push_back(2); // From here on up, no need for this kind of correction, it's done at the top + // (Add this at the end, so it is only added if we did not remove it) + } else if (type == "binary" && USEFUL_BINARY_OPS.has(node[1])) { + stack.push_back(1); + } else if ((type == "binary" && SAFE_BINARY_OPS.has(node[1])) || type == "num" || type == "name") { + stack.push_back(0); // This node is safe in that it does not interfere with this optimization + } else if (type == "unary-prefix" && node[1] == "~") { + stack.push_back(1); + } else { + stack.push_back(-1); // This node is dangerous! Give up if you see this before you see '1' + } + }; + + traversePrePost(ast, process, [&stack](Ref Node) { + stack.pop_back(); + }); + } + }; + + removeMultipleOrZero(); + return; + + // & and heap-related optimizations + + bool hasTempDoublePtr = false, rerunOrZeroPass = false; + + std::function andHeapOpts = [&hasTempDoublePtr, &rerunOrZeroPass, &andHeapOpts](Ref node) { + // Detect trees which should not + // be simplified. + Ref type = node[0]; + if (type == "sub" && node[1][0] == "name" && isFunctionTable(node[1][1])) { + return; // do not traverse subchildren here, we should not collapse 55 & 126. + } + traverseChildren(node, andHeapOpts); + // Simplifications are done now so + // that we simplify a node's operands before the node itself. This allows + // optimizations to cascade. + if (type == "name") { + if (node[1] == "tempDoublePtr") hasTempDoublePtr = true; + } else if (type == "binary" && node[1] == "&" && node[3][0] == "num") { + if (node[2][0] == "num") { + safeCopy(node, makeNum(int(node[2][1]->getNumber()) & int(node[3][1]->getNumber()))); + return; + } + Ref input = node[2]; + double amount = node[3][1]->getNumber(); + if (input[0] == "binary" && input[1] == "&" && input[3][0] == "num") { + // Collapse X & 255 & 1 + node[3][1]->setNumber(int(amount) & int(input[3][1]->getNumber())); + node[2] = input[2]; + } else if (input[0] == "sub" && input[1][0] == "name") { + // HEAP8[..] & 255 => HEAPU8[..] + HeapInfo hi = parseHeap(input[1][1]->getCString()); + if (hi.valid) { + if (isInteger32(amount) && amount == powl(2, hi.bits)-1) { + if (!hi.unsign) { + input[1][1]->setString("HEAPU" + getIntStr(hi.bits)); // make unsigned + } + // we cannot return HEAPU8 without a coercion, but at least we do HEAP8 & 255 => HEAPU8 | 0 + node[1]->setString("|"); + node[3][1]->setNumber(0); + return; + } + } + } + } else if (type == "binary" && node[1] == "^") { + // LLVM represents bitwise not as xor with -1. Translate it back to an actual bitwise not. + if (node[3][0] == "unary-prefix" && node[3][1] == "-" && node[3][2][0] == "num" && + node[3][2][1]->getNumber() == 1 && + !(node[2][0] == "unary-prefix" && node[2][1] == "~")) { // avoid creating ~~~ which is confusing for asm given the role of ~~ + safeCopy(node, make2("unary-prefix", "~", node[2])); + return; + } + } else if (type == "binary" && node[1] == ">>" && node[3][0] == "num" && + node[2][0] == "binary" && node[2][1] == "<<" && node[2][3][0] == "num" && + node[2][2][0] == "sub" && node[2][2][1][0] == "name") { + // collapse HEAPU?8[..] << 24 >> 24 etc. into HEAP8[..] | 0 + double amount = node[3][1]->getNumber(); + if (amount == node[2][3][1]->getNumber()) { + HeapInfo hi = parseHeap(node[2][2][1][1]->getCString()); + if (hi.valid && hi.bits == 32 - amount) { + node[2][2][1][1]->setString("HEAP" + getIntStr(hi.bits)); + node[1]->setString("|"); + node[2] = node[2][2]; + node[3][1]->setNumber(0); + rerunOrZeroPass = true; + return; + } + } + } else if (type == "assign") { + // optimizations for assigning into HEAP32 specifically + if (node[1]->isBool(true) && node[2][0] == "sub" && node[2][1][0] == "name") { + if (node[2][1][1] == "HEAP32") { + // HEAP32[..] = x | 0 does not need the | 0 (unless it is a mandatory |0 of a call) + if (node[3][0] == "binary" && node[3][1] == "|") { + if (node[3][2][0] == "num" && node[3][2][1]->getNumber() == 0 && node[3][3][0] != "call") { + node[3] = node[3][3]; + } else if (node[3][3][0] == "num" && node[3][3][1]->getNumber() == 0 && node[3][2][0] != "call") { + node[3] = node[3][2]; + } + } + } else if (node[2][1][1] == "HEAP8") { + // HEAP8[..] = x & 0xff does not need the & 0xff + if (node[3][0] == "binary" && node[3][1] == "&" && node[3][3][0] == "num" && node[3][3][1]->getNumber() == 0xff) { + node[3] = node[3][2]; + } + } else if (node[2][1][1] == "HEAP16") { + // HEAP16[..] = x & 0xffff does not need the & 0xffff + if (node[3][0] == "binary" && node[3][1] == "&" && node[3][3][0] == "num" && node[3][3][1]->getNumber() == 0xffff) { + node[3] = node[3][2]; + } + } + } + Ref value = node[3]; + if (value[0] == "binary" && value[1] == "|") { + // canonicalize order of |0 to end + if (value[2][0] == "num" && value[2][1]->getNumber() == 0) { + Ref temp = value[2]; + value[2] = value[3]; + value[3] = temp; + } + // if a seq ends in an |0, remove an external |0 + // note that it is only safe to do this in assigns, like we are doing here (return (x, y|0); is not valid) + if (value[2][0] == "seq" && value[2][2][0] == "binary" && USEFUL_BINARY_OPS.has(value[2][2][1])) { + node[3] = value[2]; + } + } + } else if (type == "binary" && node[1] == ">>" && node[2][0] == "num" && node[3][0] == "num") { + // optimize num >> num, in asm we need this since we do not run optimizeShifts + node[0]->setString("num"); + node[1]->setNumber(int(node[2][1]->getNumber()) >> int(node[3][1]->getNumber())); + node->setSize(2); + return; + } else if (type == "binary" && node[1] == "+") { + // The most common mathop is addition, e.g. in getelementptr done repeatedly. We can join all of those, + // by doing (num+num) ==> newnum, and (name+num)+num = name+newnum + if (node[2][0] == "num" && node[3][0] == "num") { + node[2][1]->setNumber(int(node[2][1]->getNumber()) + int(node[3][1]->getNumber())); + safeCopy(node, node[2]); + return; + } + for (int i = 2; i <= 3; i++) { + int ii = 5-i; + for (int j = 2; j <= 3; j++) { + if (node[i][0] == "num" && node[ii][0] == "binary" && node[ii][1] == "+" && node[ii][j][0] == "num") { + node[ii][j][1]->setNumber(int(node[ii][j][1]) + int(node[i][1]->getNumber())); + safeCopy(node, node[ii]); + return; + } + } + } + } + }; + traverseChildren(ast, andHeapOpts); + + if (rerunOrZeroPass) removeMultipleOrZero(); + + if (hasTempDoublePtr) { + AsmData asmData = normalizeAsm(ast); + traversePre(ast, [](Ref node) { + Ref type = node[0]; + if (type == "assign") { + if (node[1]->isBool(true) && node[2][0] == "sub" && node[2][1][0] == "name" && node[2][1][1] == "HEAP32") { + // remove bitcasts that are now obviously pointless, e.g. + // HEAP32[$45 >> 2] = HEAPF32[tempDoublePtr >> 2] = ($14 < $28 ? $14 : $28) - $42, HEAP32[tempDoublePtr >> 2] | 0; + Ref value = node[3]; + if (value[0] == "seq" && value[1][0] == "assign" && value[1][2][0] == "sub" && value[1][2][1][0] == "name" && value[1][2][1][1] == "HEAPF32" && + value[1][2][2][0] == "binary" && value[1][2][2][2][0] == "name" && value[1][2][2][2][1] == "tempDoublePtr") { + // transform to HEAPF32[$45 >> 2] = ($14 < $28 ? $14 : $28) - $42; + node[2][1][1]->setString("HEAPF32"); + node[3] = value[1][3]; + } + } + } else if (type == "seq") { + // (HEAP32[tempDoublePtr >> 2] = HEAP32[$37 >> 2], +HEAPF32[tempDoublePtr >> 2]) + // ==> + // +HEAPF32[$37 >> 2] + if (node[0] == "seq" && node[1][0] == "assign" && node[1][2][0] == "sub" && node[1][2][1][0] == "name" && + (node[1][2][1][1] == "HEAP32" || node[1][2][1][1] == "HEAPF32") && + node[1][2][2][0] == "binary" && node[1][2][2][2][0] == "name" && node[1][2][2][2][1] == "tempDoublePtr" && + node[1][3][0] == "sub" && node[1][3][1][0] == "name" && (node[1][3][1][1] == "HEAP32" || node[1][3][1][1] == "HEAPF32") && + node[2][0] != "seq") { // avoid (x, y, z) which can be used for tempDoublePtr on doubles for alignment fixes + if (node[1][2][1][1] == "HEAP32") { + node[1][3][1][1]->setString("HEAPF32"); + safeCopy(node, makeAsmCoercion(node[1][3], detectType(node[2]))); + return; + } else { + node[1][3][1][1]->setString("HEAP32"); + safeCopy(node, make3("binary", "|", node[1][3], makeNum(0))); + return; + } + } + } + }); + + // finally, wipe out remaining ones by finding cases where all assignments to X are bitcasts, and all uses are writes to + // the other heap type, then eliminate the bitcast + struct BitcastData { + int define_HEAP32, define_HEAPF32, use_HEAP32, use_HEAPF32, namings; + bool ok; + std::vector defines, uses; + + BitcastData() : define_HEAP32(0), define_HEAPF32(0), use_HEAP32(0), use_HEAPF32(0), namings(0), ok(false) {} + }; + std::map bitcastVars; + traversePre(ast, [&bitcastVars](Ref node) { + if (node[0] == "assign" && node[1]->isBool(true) && node[2][0] == "name") { + Ref value = node[3]; + if (value[0] == "seq" && value[1][0] == "assign" && value[1][2][0] == "sub" && value[1][2][1][0] == "name" && + (value[1][2][1][1] == "HEAP32" || value[1][2][1][1] == "HEAPF32") && + value[1][2][2][0] == "binary" && value[1][2][2][2][0] == "name" && value[1][2][2][2][1] == "tempDoublePtr") { + std::string name = node[2][1]->getString(); + std::string heap = value[1][2][1][1]->getString(); + if (heap == "HEAP32") { + bitcastVars[name].define_HEAP32++; + } else { + assert(heap == "HEAPF32"); + bitcastVars[name].define_HEAPF32++; + } + bitcastVars[name].defines.push_back(node); + bitcastVars[name].ok = true; + } + } + }); + traversePre(ast, [&bitcastVars](Ref node) { + Ref type = node[0]; + if (type == "name" && bitcastVars[node[1]->getString()].ok) { + bitcastVars[node[1]->getString()].namings++; + } else if (type == "assign" && node[1]->isBool(true)) { + Ref value = node[3]; + if (value[0] == "name") { + std::string name = value[1]->getString(); + if (bitcastVars[name].ok) { + Ref target = node[2]; + if (target[0] == "sub" && target[1][0] == "name" && (target[1][1] == "HEAP32" || target[1][1] == "HEAPF32")) { + if (target[1][1] == "HEAP32") { + bitcastVars[name].use_HEAP32++; + } else { + bitcastVars[name].use_HEAPF32++; + } + bitcastVars[name].uses.push_back(node); + } + } + } + } + }); + for (auto iter : bitcastVars) { + const std::string& v = iter.first; + BitcastData& info = iter.second; + // good variables define only one type, use only one type, have definitions and uses, and define as a different type than they use + if (info.define_HEAP32*info.define_HEAPF32 == 0 && info.use_HEAP32*info.use_HEAPF32 == 0 && + info.define_HEAP32+info.define_HEAPF32 > 0 && info.use_HEAP32+info.use_HEAPF32 > 0 && + info.define_HEAP32*info.use_HEAP32 == 0 && info.define_HEAPF32*info.use_HEAPF32 == 0 && + asmData.types.count(v) && info.namings == info.define_HEAP32+info.define_HEAPF32+info.use_HEAP32+info.use_HEAPF32) { + std::string correct = info.use_HEAP32 ? "HEAPF32" : "HEAP32"; + for (auto define : info.defines) { + define[3] = define[3][1][3]; + if (correct == "HEAP32") { + define[3] = make3("binary", "|", define[3], makeNum(0)); + } else { + define[3] = makeAsmCoercion(define[3], preciseF32 ? ASM_FLOAT : ASM_DOUBLE); + } + // do we want a simplifybitops on the new values here? + } + for (auto use : info.uses) { + use[2][1][1]->setString(correct); + } + AsmType correctType; + switch(asmData.types[v]) { + case ASM_INT: correctType = preciseF32 ? ASM_FLOAT : ASM_DOUBLE; break; + case ASM_FLOAT: case ASM_DOUBLE: correctType = ASM_INT; break; + default: {} // pass + } + asmData.types[v] = correctType; + } + } + denormalizeAsm(ast, asmData); + } + }; + + std::function emitsBoolean = [&emitsBoolean](Ref node) { + std::string& type = node[0]->getString(); + if (type == "num") { + return node[1]->getNumber() == 0 || node[1] == 1; + } + if (type == "binary") return COMPARE_OPS.has(node[1]); + if (type == "unary-prefix") return node[1] == "!"; + if (type == "conditional") return emitsBoolean(node[2]) && emitsBoolean(node[3]); + return false; + }; + + // expensive | expensive can be turned into expensive ? 1 : expensive, and + // expensive | cheap can be turned into cheap ? 1 : expensive, + // so that we can avoid the expensive computation, if it has no side effects. + auto conditionalize = [&emitsBoolean](Ref ast) { + const int MIN_COST = 7; + traversePre(ast, [&emitsBoolean](Ref node) { + if (node[0] == "binary" && (node[1] == "|" || node[1] == "&") && node[3][0] != "num" && node[2][0] != "num") { + // logical operator on two non-numerical values + Ref left = node[2]; + Ref right = node[3]; + if (!emitsBoolean(left) || !emitsBoolean(right)) return; + bool leftEffects = hasSideEffects(left); + bool rightEffects = hasSideEffects(right); + if (leftEffects && rightEffects) return; // both must execute + // canonicalize with side effects, if any, happening on the left + if (rightEffects) { + if (measureCost(left) < MIN_COST) return; // avoidable code is too cheap + Ref temp = left; + left = right; + right = temp; + } else if (leftEffects) { + if (measureCost(right) < MIN_COST) return; // avoidable code is too cheap + } else { + // no side effects, reorder based on cost estimation + int leftCost = measureCost(left); + int rightCost = measureCost(right); + if (std::max(leftCost, rightCost) < MIN_COST) return; // avoidable code is too cheap + // canonicalize with expensive code on the right + if (leftCost > rightCost) { + Ref temp = left; + left = right; + right = temp; + } + } + // worth it, perform conditionalization + Ref ret; + if (node[1] == "|") { + ret = make3("conditional", left, makeNum(1), right); + } else { // & + ret = make3("conditional", left, right, makeNum(0)); + } + if (left[0] == "unary-prefix" && left[1] == "!") { + ret[1] = flipCondition(left); + Ref temp = ret[2]; + ret[2] = ret[3]; + ret[3] = temp; + } + safeCopy(node, ret); + return; + } + }); + }; + + traverseFunctions(ast, [&](Ref func) { + //simplifyIntegerConversions(func); + simplifyOps(func); + //simplifyNotComps(func); // XXX broken in JS, should be a traverse, call that, assign on the node if return is different than itself + //conditionalize(func); + }); +} + void simplifyIfs(Ref ast) { traverseFunctions(ast, [](Ref func) { bool simplifiedAnElse = false; - traversePre(func, [&simplifiedAnElse](Ref node) { + traversePre(func, [&simplifiedAnElse](Ref node) { // simplify if (x) { if (y) { .. } } to if (x ? y : 0) { .. } if (node[0] == "if") { Ref body = node[2]; @@ -411,7 +1185,7 @@ void simplifyIfs(Ref ast) { std::unordered_map labelAssigns; - traversePre(func, [&labelAssigns, &abort](Ref node) { + traversePre(func, [&labelAssigns, &abort](Ref node) { if (node[0] == "assign" && node[2][0] == "name" && node[2][1] == "label") { if (node[3][0] == "num") { int value = node[3][1]->getNumber(); @@ -426,7 +1200,7 @@ void simplifyIfs(Ref ast) { std::unordered_map labelChecks; - traversePre(func, [&labelChecks, &abort](Ref node) { + traversePre(func, [&labelChecks, &abort](Ref node) { if (node[0] == "binary" && node[1] == "==" && node[2][0] == "binary" && node[2][1] == "|" && node[2][2][0] == "name" && node[2][2][1] == "label") { if (node[3][0] == "num") { @@ -441,7 +1215,7 @@ void simplifyIfs(Ref ast) { if (abort) return; int inLoop = 0; // when in a loop, we do not emit label = 0; in the relooper as there is no need - traversePrePost(func, [&inLoop, &labelAssigns, &labelChecks](Ref node) { + traversePrePost(func, [&inLoop, &labelAssigns, &labelChecks](Ref node) { if (node[0] == "while") inLoop++; Ref stats = getStatements(node); if (!stats->isNull() && stats->size() > 0) { @@ -482,7 +1256,7 @@ void simplifyIfs(Ref ast) { } } } - }, [&inLoop](Ref node) { + }, [&inLoop](Ref node) { if (node[0] == "while") inLoop--; }); assert(inLoop == 0); @@ -494,7 +1268,7 @@ void optimizeFrounds(Ref ast) { // collapse fround(fround(..)), which can happen due to elimination // also emit f0 instead of fround(0) (except in returns) bool inReturn = false; - std::function fix = [&](Ref node) { + std::function fix = [&](Ref node) { bool ret = node[0] == "return"; if (ret) inReturn = true; traverseChildren(node, fix); @@ -504,7 +1278,7 @@ void optimizeFrounds(Ref ast) { if (arg[0] == "num") { if (!inReturn && arg[1]->getNumber() == 0) *node = *makeName("f0"); } else if (arg[0] == "call" && arg[1][0] == "name" && arg[1][1] == "Math_fround") { - *node = *arg; + safeCopy(node, arg); } } }; @@ -544,6 +1318,7 @@ int main(int argc, char **argv) { if (str == "asm") {} // the default for us else if (str == "asmPreciseF32") preciseF32 = true; else if (str == "receiveJSON" || str == "emitJSON") {} // the default for us + else if (str == "simplifyExpressions") simplifyExpressions(doc); else if (str == "optimizeFrounds") optimizeFrounds(doc); else if (str == "simplifyIfs") simplifyIfs(doc); else { From f54abf922b8e3da751eed708c4e202c03f5ed9e9 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 5 Nov 2014 17:40:38 -0800 Subject: [PATCH 042/161] enable more parts of simplifyExpressions, and use Value == when comparing Refs --- tools/optimizer/minijson.h | 17 +++++++++++++++++ tools/optimizer/optimizer.cpp | 15 ++++----------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index fb903147a4e8c..6bdeabac10b18 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -12,14 +12,18 @@ #include #include #include +#include #include #include #define err(str) fprintf(stderr, str "\n"); #define errv(str, ...) fprintf(stderr, str "\n", __VA_ARGS__); +class Ref; struct Value; +void dump(const char *str, Ref node, bool pretty=false); + // Reference to a value. Simple shared_ptr, plus [] operator for convenience - we work on lots of arrays class Ref : public std::shared_ptr { public: @@ -36,6 +40,7 @@ class Ref : public std::shared_ptr { bool operator==(const char *str); // comparison to string, which is by value bool operator!=(const char *str); bool operator==(double d) { assert(0); } // prevent Ref == number, which is potentially ambiguous; use ->getNumber() == number + bool operator==(Ref other); bool operator!(); // check if null, in effect }; @@ -387,7 +392,19 @@ bool Ref::operator!=(const char *str) { return get()->isString() ? get()->getString() != str : true; } +bool Ref::operator==(Ref other) { + return **this == *other; +} + bool Ref::operator!() { return get()->isNull(); } +// dump + +void dump(const char *str, Ref node, bool pretty) { + std::cerr << str << ": "; + node->stringify(std::cerr, pretty); + std::cerr << std::endl; +} + diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index c1903e73a279c..4710f045fa6ae 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -8,7 +7,6 @@ #include #include #include -#include #include "minijson.h" @@ -22,12 +20,6 @@ Ref doc; // Infrastructure //================== -void dump(const char *str, Ref node, bool pretty=false) { - std::cerr << str << ": "; - node->stringify(std::cerr, pretty); - std::cerr << std::endl; -} - int parseInt(const char *str) { int ret = *str - '0'; while (*(++str)) { @@ -448,7 +440,8 @@ AsmData normalizeAsm(Ref func) { if (node[0] != "stat" || node[1][0] != "assign" || node[1][2][0] != "name") break; node = node[1]; Ref name = node[2][1]; - if (!!func[2] && func[2]->indexOf(name) < 0) break; // not an assign into a parameter, but a global + int index = func[2]->indexOf(name); + if (index < 0) break; // not an assign into a parameter, but a global std::string& str = name->getString(); if (data.params.count(str) > 0) break; // already done that param, must be starting function body data.params[str] = func[2]->indexOf(name); @@ -477,9 +470,10 @@ AsmData normalizeAsm(Ref func) { outside: // look for other var definitions and collect them while (i < stats->size()) { - traversePre(stats[i], [](Ref node) { + traversePre(stats[i], [&](Ref node) { Ref type = node[0]; if (type == "var") { + dump("bad, seeing a var in need of fixing", func); assert(0); //, 'should be no vars to fix! ' + func[1] + ' : ' + JSON.stringify(node)); } }); @@ -776,7 +770,6 @@ void simplifyExpressions(Ref ast) { }; removeMultipleOrZero(); - return; // & and heap-related optimizations From 12a8cd94430c08026878ab96f95331e759084b4d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 5 Nov 2014 17:47:05 -0800 Subject: [PATCH 043/161] enable more parts of simplifyExpressions --- tools/optimizer/optimizer.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 4710f045fa6ae..f4d92959d843b 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -669,7 +669,7 @@ void simplifyExpressions(Ref ast) { auto simplifyIntegerConversions = [](Ref ast) { traversePre(ast, [](Ref node) { Ref type = node[0]; - if (type == "binary" && node[1] == ">>" && node[3][0] == "num" && + if (type == "binary" && node[1] == ">>" && node[3][0] == "num" && node[2][0] == "binary" && node[2][1] == "<<" && node[2][3][0] == "num" && node[3][1]->getNumber() == node[2][3][1]->getNumber()) { // Transform (x&A)<>B to X&A. Ref innerNode = node[2][2]; @@ -1032,7 +1032,7 @@ void simplifyExpressions(Ref ast) { std::function emitsBoolean = [&emitsBoolean](Ref node) { std::string& type = node[0]->getString(); if (type == "num") { - return node[1]->getNumber() == 0 || node[1] == 1; + return node[1]->getNumber() == 0 || node[1]->getNumber() == 1; } if (type == "binary") return COMPARE_OPS.has(node[1]); if (type == "unary-prefix") return node[1] == "!"; @@ -1094,10 +1094,10 @@ void simplifyExpressions(Ref ast) { }; traverseFunctions(ast, [&](Ref func) { - //simplifyIntegerConversions(func); + simplifyIntegerConversions(func); simplifyOps(func); //simplifyNotComps(func); // XXX broken in JS, should be a traverse, call that, assign on the node if return is different than itself - //conditionalize(func); + conditionalize(func); }); } From b3973ded6c61914072f68b90e0a0b02db9e05420 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 5 Nov 2014 17:51:19 -0800 Subject: [PATCH 044/161] enable simplifyNotComps --- tools/optimizer/optimizer.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index f4d92959d843b..f8146327c0336 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -1096,7 +1096,12 @@ void simplifyExpressions(Ref ast) { traverseFunctions(ast, [&](Ref func) { simplifyIntegerConversions(func); simplifyOps(func); - //simplifyNotComps(func); // XXX broken in JS, should be a traverse, call that, assign on the node if return is different than itself + traversePre(func, [](Ref node) { + Ref ret = simplifyNotCompsDirect(node); + if (ret.get() != node.get()) { // if we received a different pointer in return, then we need to copy the new value + safeCopy(node, ret); + } + }); conditionalize(func); }); } From e703658c7eada1410c5c347aad4890aea9aef711 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 5 Nov 2014 17:54:33 -0800 Subject: [PATCH 045/161] do not emit params as vars --- tools/optimizer/optimizer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index f8146327c0336..2ccb83a46a7ee 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -500,7 +500,7 @@ void denormalizeAsm(Ref func, AsmData& data) { // calculate variable definitions Ref varDefs = new ArrayValue(); for (auto v : data.types) { - varDefs->push_back(makeAsmVarDef(v.first, v.second)); + if (data.params.count(v.first) == 0) varDefs->push_back(makeAsmVarDef(v.first, v.second)); } // each param needs a line; reuse emptyNodes as much as we can int numParams = data.params.size(); From 3442f0c542022b185f14a81e5ba9147ed6bcf33c Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 5 Nov 2014 18:02:50 -0800 Subject: [PATCH 046/161] avoid strncmp --- tools/optimizer/optimizer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 2ccb83a46a7ee..5623950c65e99 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -140,7 +140,7 @@ struct HeapInfo { HeapInfo parseHeap(const char *name) { HeapInfo ret; - if (!strncmp(name, "HEAP", 4)) { + if (name[0] != 'H' || name[1] != 'E' || name[2] != 'A' || name[3] != 'P') { ret.valid = false; return ret; } From 3559bdc9eba5ca462f5ff4d0849d21fe372abc99 Mon Sep 17 00:00:00 2001 From: Peter van der Tak Date: Thu, 6 Nov 2014 12:04:09 +0100 Subject: [PATCH 047/161] Fix invoking a javascript function contained in an emscripten::val when compiled with closure. Closure translates argPackAdvance to a small variable name, but does not translate 'argPackAdvance'. --- src/embind/emval.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/embind/emval.js b/src/embind/emval.js index 84af9870f4e44..466a9df1f679a 100644 --- a/src/embind/emval.js +++ b/src/embind/emval.js @@ -159,7 +159,7 @@ var LibraryEmVal = { functionBody += "var argType"+i+" = requireRegisteredType(HEAP32[(argTypes >> 2) + "+i+"], \"parameter "+i+"\");\n" + "var arg"+i+" = argType"+i+".readValueFromPointer(args);\n" + - "args += argType"+i+".argPackAdvance;\n"; + "args += argType"+i+"['argPackAdvance'];\n"; } functionBody += "var obj = new constructor("+argsList+");\n" + @@ -246,7 +246,7 @@ var LibraryEmVal = { for (var i = 0; i < argCount; ++i) { var type = types[i]; args[i] = type['readValueFromPointer'](argv); - argv += type.argPackAdvance; + argv += type['argPackAdvance']; } var rv = handle.apply(undefined, args); @@ -307,7 +307,7 @@ var LibraryEmVal = { for (var i = 0; i < argCount - 1; ++i) { functionBody += " var arg" + i + " = argType" + i + ".readValueFromPointer(args" + (offset ? ("+"+offset) : "") + ");\n"; - offset += types[i + 1].argPackAdvance; + offset += types[i + 1]['argPackAdvance']; } functionBody += " var rv = handle[name](" + argsList + ");\n"; From 54e16823d74a58b1ecdda044f07b0cdda4afa859 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 6 Nov 2014 10:52:05 -0800 Subject: [PATCH 048/161] refactor AsmData --- tools/optimizer/optimizer.cpp | 302 ++++++++++++++++++---------------- 1 file changed, 160 insertions(+), 142 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 5623950c65e99..37ad84794295a 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -118,18 +118,166 @@ enum AsmType { ASM_NONE = 5 }; +// forward decls +struct AsmData; +AsmType detectType(Ref node, AsmData *asmData=nullptr, bool inVarDef=false); +Ref makeEmpty(); +bool isEmpty(Ref node); +Ref makeAsmVarDef(const std::string& v_, AsmType type); +Ref makeNum(double x); +Ref makeName(const std::string& str); +Ref makeAsmCoercion(Ref node, AsmType type); +Ref make1(const char* type, Ref a); +Ref make3(const char *type, Ref a, Ref b, Ref c); + struct AsmData { - std::map params; // name => index - std::map types; // name => type + struct Local { + AsmType type; + bool param; // false if a var + }; + typedef std::map Locals; + + Locals locals; + std::vector params; // in order + std::vector vars; // in order AsmType ret; - AsmData() : ret(ASM_NONE) {} + Ref func; - AsmType getType(std::string name) { - auto ret = types.find(name); - if (ret != types.end()) return ret->second; + AsmType getType(const std::string& name) { + auto ret = locals.find(name); + if (ret != locals.end()) return ret->second.type; return ASM_NONE; } + void setType(const std::string& name, AsmType type) { + locals[name].type = type; + } + + bool isLocal(const std::string& name) { + return locals.count(name) > 0; + } + + AsmData(Ref f) { + func = f; + + // process initial params + Ref stats = func[3]; + int i = 0; + while (i < stats->size()) { + Ref node = stats[i]; + if (node[0] != "stat" || node[1][0] != "assign" || node[1][2][0] != "name") break; + node = node[1]; + Ref name = node[2][1]; + int index = func[2]->indexOf(name); + if (index < 0) break; // not an assign into a parameter, but a global + std::string& str = name->getString(); + if (locals.count(str) > 0) break; // already done that param, must be starting function body + locals[str] = { detectType(node[3]), true }; + params.push_back(locals.find(str)); + stats[i] = makeEmpty(); + i++; + } + // process initial variable definitions + while (i < stats->size()) { + Ref node = stats[i]; + if (node[0] != "var") break; + for (int j = 0; j < node[1]->size(); j++) { + Ref v = node[1][j]; + std::string name = v[0]->getString(); + Ref value = v[1]; + if (locals.count(name) == 0) { + locals[name] = { detectType(value, nullptr, true), false }; + vars.push_back(locals.find(name)); + v->setSize(1); // make an un-assigning var + } else { + assert(j == 0); // cannot break in the middle + goto outside; + } + } + i++; + } + outside: + // look for other var definitions and collect them + while (i < stats->size()) { + traversePre(stats[i], [&](Ref node) { + Ref type = node[0]; + if (type == "var") { + dump("bad, seeing a var in need of fixing", func); + assert(0); //, 'should be no vars to fix! ' + func[1] + ' : ' + JSON.stringify(node)); + } + }); + i++; + } + // look for final "return" statement to get return type. + Ref retStmt = stats[stats->size() - 1]; + if (!!retStmt && retStmt[0] == "return" && !!retStmt[1]) { + ret = detectType(retStmt[1]); + } else { + ret = ASM_NONE; + } + } + + void denormalize() { + Ref stats = func[3]; + // Remove var definitions, if any + for (int i = 0; i < stats->size(); i++) { + if (stats[i][0] == "var") { + stats[i] = makeEmpty(); + } else { + if (!isEmpty(stats[i])) break; + } + } + // calculate variable definitions + Ref varDefs = new ArrayValue(); + for (auto v : vars) { + varDefs->push_back(makeAsmVarDef(v->first, v->second.type)); + } + // each param needs a line; reuse emptyNodes as much as we can + int numParams = params.size(); + int emptyNodes = 0; + while (emptyNodes < stats->size()) { + if (!isEmpty(stats[emptyNodes])) break; + emptyNodes++; + } + int neededEmptyNodes = numParams + (varDefs->size() ? 1 : 0); // params plus one big var if there are vars + if (neededEmptyNodes > emptyNodes) { + stats->insert(0, neededEmptyNodes - emptyNodes); + } else if (neededEmptyNodes < emptyNodes) { + stats->splice(0, emptyNodes - neededEmptyNodes); + } + // add param coercions + int next = 0; + for (auto param : func[2]->getArray()) { + std::string& str = param->getString(); + stats[next++] = make1("stat", make3("assign", &(new Value())->setBool(true), makeName(str), makeAsmCoercion(makeName(str), locals[str].type))); + } + if (varDefs->size()) { + stats[next] = make1("var", varDefs); + } + /* + if (inlines.length > 0) { + var i = 0; + traverse(func, function(node, type) { + if (type == "call" && node[1][0] == "name" && node[1][1] == 'inlinejs') { + node[1] = inlines[i++]; // swap back in the body + } + }); + } + */ + // ensure that there's a final "return" statement if needed. + if (ret != ASM_NONE) { + Ref retStmt = stats[stats->size() - 1]; + if (!retStmt || retStmt[0]->getString() != "return") { + Ref retVal = makeNum(0); + if (ret != ASM_INT) { + retVal = makeAsmCoercion(retVal, ret); + } + stats->push_back(make1("return", retVal)); + } + } + //printErr('denormalized \n\n' + astToSrc(func) + '\n\n'); + } + }; struct HeapInfo { @@ -162,7 +310,7 @@ bool isInteger32(double x) { std::string ASM_FLOAT_ZERO; -AsmType detectType(Ref node, AsmData *asmData=nullptr, bool inVarDef=false) { +AsmType detectType(Ref node, AsmData *asmData, bool inVarDef) { switch (node[0]->getString()[0]) { case 'n': { if (node[0] == "num") { @@ -417,136 +565,6 @@ bool hasSideEffects(Ref node) { // this is 99% incomplete! } // Transforms -/* -struct AsmData { - std::map params; // name => index - std::map types; // name => type - type ret - - AsmType getType(std::string name) { - auto ret = types.find(name); - if (ret != types.end()) return ret->second; - return ASM_NONE; - } -}; -*/ -AsmData normalizeAsm(Ref func) { - AsmData data; - // process initial params - Ref stats = func[3]; - int i = 0; - while (i < stats->size()) { - Ref node = stats[i]; - if (node[0] != "stat" || node[1][0] != "assign" || node[1][2][0] != "name") break; - node = node[1]; - Ref name = node[2][1]; - int index = func[2]->indexOf(name); - if (index < 0) break; // not an assign into a parameter, but a global - std::string& str = name->getString(); - if (data.params.count(str) > 0) break; // already done that param, must be starting function body - data.params[str] = func[2]->indexOf(name); - data.types[str] = detectType(node[3]); - stats[i] = makeEmpty(); - i++; - } - // process initial variable definitions - while (i < stats->size()) { - Ref node = stats[i]; - if (node[0] != "var") break; - for (int j = 0; j < node[1]->size(); j++) { - Ref v = node[1][j]; - std::string name = v[0]->getString(); - Ref value = v[1]; - if (!data.types.count(name)) { - data.types[name] = detectType(value, nullptr, true); - v->setSize(1); // make an un-assigning var - } else { - assert(j == 0); // cannot break in the middle - goto outside; - } - } - i++; - } - outside: - // look for other var definitions and collect them - while (i < stats->size()) { - traversePre(stats[i], [&](Ref node) { - Ref type = node[0]; - if (type == "var") { - dump("bad, seeing a var in need of fixing", func); - assert(0); //, 'should be no vars to fix! ' + func[1] + ' : ' + JSON.stringify(node)); - } - }); - i++; - } - // look for final "return" statement to get return type. - Ref retStmt = stats[stats->size() - 1]; - if (!!retStmt && retStmt[0] == "return" && !!retStmt[1]) { - data.ret = detectType(retStmt[1]); - } - return data; -} - -void denormalizeAsm(Ref func, AsmData& data) { - Ref stats = func[3]; - // Remove var definitions, if any - for (int i = 0; i < stats->size(); i++) { - if (stats[i][0] == "var") { - stats[i] = makeEmpty(); - } else { - if (!isEmpty(stats[i])) break; - } - } - // calculate variable definitions - Ref varDefs = new ArrayValue(); - for (auto v : data.types) { - if (data.params.count(v.first) == 0) varDefs->push_back(makeAsmVarDef(v.first, v.second)); - } - // each param needs a line; reuse emptyNodes as much as we can - int numParams = data.params.size(); - int emptyNodes = 0; - while (emptyNodes < stats->size()) { - if (!isEmpty(stats[emptyNodes])) break; - emptyNodes++; - } - int neededEmptyNodes = numParams + (varDefs->size() ? 1 : 0); // params plus one big var if there are vars - if (neededEmptyNodes > emptyNodes) { - stats->insert(0, neededEmptyNodes - emptyNodes); - } else if (neededEmptyNodes < emptyNodes) { - stats->splice(0, emptyNodes - neededEmptyNodes); - } - // add param coercions - int next = 0; - for (auto param : func[2]->getArray()) { - std::string& str = param->getString(); - stats[next++] = make1("stat", make3("assign", &(new Value())->setBool(true), makeName(str), makeAsmCoercion(makeName(str), data.types[str]))); - } - if (varDefs->size()) { - stats[next] = make1("var", varDefs); - } - /* - if (data.inlines.length > 0) { - var i = 0; - traverse(func, function(node, type) { - if (type == "call" && node[1][0] == "name" && node[1][1] == 'inlinejs') { - node[1] = data.inlines[i++]; // swap back in the body - } - }); - } - */ - // ensure that there's a final "return" statement if needed. - if (data.ret != ASM_NONE) { - Ref retStmt = stats[stats->size() - 1]; - if (!retStmt || retStmt[0]->getString() != "return") { - Ref retVal = makeNum(0); - if (data.ret != ASM_INT) { - retVal = makeAsmCoercion(retVal, data.ret); - } - stats->push_back(make1("return", retVal)); - } - } - //printErr('denormalized \n\n' + astToSrc(func) + '\n\n'); -} // We often have branchings that are simplified so one end vanishes, and // we then get @@ -907,7 +925,7 @@ void simplifyExpressions(Ref ast) { if (rerunOrZeroPass) removeMultipleOrZero(); if (hasTempDoublePtr) { - AsmData asmData = normalizeAsm(ast); + AsmData asmData(ast); traversePre(ast, [](Ref node) { Ref type = node[0]; if (type == "assign") { @@ -1002,7 +1020,7 @@ void simplifyExpressions(Ref ast) { if (info.define_HEAP32*info.define_HEAPF32 == 0 && info.use_HEAP32*info.use_HEAPF32 == 0 && info.define_HEAP32+info.define_HEAPF32 > 0 && info.use_HEAP32+info.use_HEAPF32 > 0 && info.define_HEAP32*info.use_HEAP32 == 0 && info.define_HEAPF32*info.use_HEAPF32 == 0 && - asmData.types.count(v) && info.namings == info.define_HEAP32+info.define_HEAPF32+info.use_HEAP32+info.use_HEAPF32) { + asmData.isLocal(v) && info.namings == info.define_HEAP32+info.define_HEAPF32+info.use_HEAP32+info.use_HEAPF32) { std::string correct = info.use_HEAP32 ? "HEAPF32" : "HEAP32"; for (auto define : info.defines) { define[3] = define[3][1][3]; @@ -1017,15 +1035,15 @@ void simplifyExpressions(Ref ast) { use[2][1][1]->setString(correct); } AsmType correctType; - switch(asmData.types[v]) { + switch(asmData.getType(v)) { case ASM_INT: correctType = preciseF32 ? ASM_FLOAT : ASM_DOUBLE; break; case ASM_FLOAT: case ASM_DOUBLE: correctType = ASM_INT; break; default: {} // pass } - asmData.types[v] = correctType; + asmData.setType(v, correctType); } } - denormalizeAsm(ast, asmData); + asmData.denormalize(); } }; From 19340e803b6f265fd279593927c5f8da35b97110 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 6 Nov 2014 11:51:11 -0800 Subject: [PATCH 049/161] fix fround var defs --- tools/js_optimizer.py | 2 +- tools/optimizer/optimizer.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index b97c02b4e39d9..98c7d2726db47 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -106,7 +106,7 @@ def run_on_chunk(command): filename = command[index + 1] else: filename = command[1] - #print >> sys.stderr, 'running js optimizer command', command, filename + #print >> sys.stderr, 'running js optimizer command', ' '.join(map(lambda c: c if c != filename else 'input.txt', command)) #shutil.copyfile(filename, '/tmp/emscripten_temp/input.txt') output = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0] assert len(output) > 0 and not output.startswith('Assertion failed'), 'Error in js optimizer: ' + output diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 37ad84794295a..eacf0e3c7520b 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -479,7 +479,7 @@ Ref makeAsmVarDef(const std::string& v_, AsmType type) { if (ASM_FLOAT_ZERO.size() > 0) { val = makeName(ASM_FLOAT_ZERO.c_str()); } else { - return make2("call", makeName("Math_fround"), &(new ArrayValue())->push_back(makeNum(0))); + val = make2("call", makeName("Math_fround"), &(new ArrayValue())->push_back(makeNum(0))); } break; } From 55c7bd0a6fc710e98466225d6876f36132275f9b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 6 Nov 2014 13:15:28 -0800 Subject: [PATCH 050/161] emitJSON/receiveJSON fixes --- tools/js_optimizer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 98c7d2726db47..bfdb4fa197cc2 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -293,7 +293,7 @@ def write_chunk(chunk, i): filenames = [] if len(filenames) > 0: - if not use_native(passes, source_map): + if not use_native(passes, source_map) or 'receiveJSON' not in passes or 'emitJSON' not in passes: commands = map(lambda filename: js_engine + [JS_OPTIMIZER, filename, 'noPrintMetadata'] + (['--debug'] if source_map else []) + passes, filenames) @@ -403,6 +403,8 @@ def sorter(x, y): return filename def run(filename, passes, js_engine=shared.NODE_JS, jcache=False, source_map=False, extra_info=None, just_split=False, just_concat=False): + if 'receiveJSON' in passes: just_split = True + if 'emitJSON' in passes: just_concat = True js_engine = shared.listify(js_engine) return temp_files.run_and_clean(lambda: run_on_js(filename, passes, js_engine, jcache, source_map, extra_info, just_split, just_concat)) From d85213d9cc67425c3b30f90c38de1881824fd71e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 6 Nov 2014 13:35:37 -0800 Subject: [PATCH 051/161] fix missing getNumber bug --- tools/optimizer/optimizer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index eacf0e3c7520b..9fdf6dc0d3243 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -912,7 +912,7 @@ void simplifyExpressions(Ref ast) { int ii = 5-i; for (int j = 2; j <= 3; j++) { if (node[i][0] == "num" && node[ii][0] == "binary" && node[ii][1] == "+" && node[ii][j][0] == "num") { - node[ii][j][1]->setNumber(int(node[ii][j][1]) + int(node[i][1]->getNumber())); + node[ii][j][1]->setNumber(int(node[ii][j][1]->getNumber()) + int(node[i][1]->getNumber())); safeCopy(node, node[ii]); return; } From 61ccd106c3121ab5a56b4f8028e4cc45488f6501 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Fri, 7 Nov 2014 00:13:40 +0100 Subject: [PATCH 052/161] no-eval fallback for cwrap cwrap returns a dynamically generated function for better performances. However, in some environments, calling eval is not possible. So this patch proposes a static version of cwrap when NO_DYNAMIC_EXECUTION is activated. --- src/preamble.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index a1bc48ecefedf..68a110cbea402 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -391,6 +391,7 @@ var cwrap, ccall; return ret; } +#if NO_DYNAMIC_EXECUTION == 0 var sourceRegex = /^function\s*\(([^)]*)\)\s*{\s*([^*]*?)[\s;]*(?:return\s*(.*?)[;\s]*)?}$/; function parseJSFunc(jsfunc) { // Match the body and the return value of a javascript function source @@ -449,12 +450,17 @@ var cwrap, ccall; funcstr += JSsource['stackRestore'].body + ';'; } funcstr += 'return ret})'; -#if NO_DYNAMIC_EXECUTION == 0 return eval(funcstr); + }; #else - abort('NO_DYNAMIC_EXECUTION was set, cannot eval - ccall is not functional'); + // NO_DYNAMIC_EXECUTION is on, so we can't use the fast version of cwrap. + // Fall back to returning a bound version of ccall. + cwrap = function cwrap(ident, returnType, argTypes) { + return function() { + return ccall(ident, returnType, argTypes, arguments); + } + } #endif - }; })(); Module["cwrap"] = cwrap; Module["ccall"] = ccall; From a3298c62056b8ce55f3e0952b160d526ebe02885 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 6 Nov 2014 18:34:15 -0800 Subject: [PATCH 053/161] update optimizer build flags --- tools/js_optimizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index bfdb4fa197cc2..87d79779da522 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -29,7 +29,7 @@ def create_optimizer(): shared.logging.debug('building native optimizer') output = shared.Cache.get_path('optimizer.exe') shared.try_delete(output) - subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-I' + shared.path_from_root('tools', 'optimizer', 'rapidjson'), '-O2', '-std=c++11', '-o', output, '-g']).communicate() + subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-O3', '-std=c++11', '-o', output]).communicate() assert os.path.exists(output) return output return shared.Cache.get('optimizer.exe', create_optimizer, extension='exe') From 46f562111542a8431836e0e6abb30fc273abf36a Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 6 Nov 2014 20:32:36 -0800 Subject: [PATCH 054/161] comment about debugging/profiling flags --- tools/js_optimizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 87d79779da522..9c1e64c5a4525 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -29,7 +29,7 @@ def create_optimizer(): shared.logging.debug('building native optimizer') output = shared.Cache.get_path('optimizer.exe') shared.try_delete(output) - subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-O3', '-std=c++11', '-o', output]).communicate() + subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-O3', '-std=c++11', '-o', output]).communicate() # , '-g', '-fno-omit-frame-pointer' assert os.path.exists(output) return output return shared.Cache.get('optimizer.exe', create_optimizer, extension='exe') From d6d4a81839105bf46be7d10746df737894c5549c Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 6 Nov 2014 20:32:54 -0800 Subject: [PATCH 055/161] use unordered_map --- tools/optimizer/minijson.h | 2 ++ tools/optimizer/optimizer.cpp | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 6bdeabac10b18..5974c28168338 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -4,6 +4,8 @@ * Uses shared_ptr for simplicity, basically everywhere. TODO: measure impact */ +//#define NDEBUG + #include #include #include diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 9fdf6dc0d3243..c25b618008de2 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include @@ -135,7 +134,7 @@ struct AsmData { AsmType type; bool param; // false if a var }; - typedef std::map Locals; + typedef std::unordered_map Locals; Locals locals; std::vector params; // in order @@ -971,7 +970,7 @@ void simplifyExpressions(Ref ast) { BitcastData() : define_HEAP32(0), define_HEAPF32(0), use_HEAP32(0), use_HEAPF32(0), namings(0), ok(false) {} }; - std::map bitcastVars; + std::unordered_map bitcastVars; traversePre(ast, [&bitcastVars](Ref node) { if (node[0] == "assign" && node[1]->isBool(true) && node[2][0] == "name") { Ref value = node[3]; From 51a594d9d268a66ebef1e6b55eea552fa0023f15 Mon Sep 17 00:00:00 2001 From: Peter van der Tak Date: Fri, 7 Nov 2014 15:17:29 +0100 Subject: [PATCH 056/161] Add failing test case for returning unique_ptr with embind. A unique_ptr returned to javascript will call the destructor immediately. Calling .delete() on the returned object from javascript calls the destructor a second time. --- tests/embind/embind.test.js | 15 +++++++++++++++ tests/embind/embind_test.cpp | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/tests/embind/embind.test.js b/tests/embind/embind.test.js index 192e2e904cd44..bb373eaea5971 100644 --- a/tests/embind/embind.test.js +++ b/tests/embind/embind.test.js @@ -1297,6 +1297,21 @@ module({ }); }); + test("returned unique_ptr does not call destructor", function() { + var logged = ""; + var c = new cm.emval_test_return_unique_ptr_lifetime(function (s) { logged += s; }); + assert.equal("(constructor)", logged); + c.delete(); + }); + + test("returned unique_ptr calls destructor on delete", function() { + var logged = ""; + var c = new cm.emval_test_return_unique_ptr_lifetime(function (s) { logged += s; }); + logged = ""; + c.delete(); + assert.equal("(destructor)", logged); + }); + test("StringHolder", function() { var a = new cm.StringHolder("foobar"); assert.equal("foobar", a.get()); diff --git a/tests/embind/embind_test.cpp b/tests/embind/embind_test.cpp index c274a0f111f89..b8cf90b79a275 100644 --- a/tests/embind/embind_test.cpp +++ b/tests/embind/embind_test.cpp @@ -910,6 +910,23 @@ std::unique_ptr emval_test_return_unique_ptr() { return std::unique_ptr(new ValHolder(val::object())); } +class UniquePtrLifetimeMock { +public: + UniquePtrLifetimeMock(val l) : logger(l) { + logger(std::string("(constructor)")); + } + ~UniquePtrLifetimeMock() { + logger(std::string("(destructor)")); + } + +private: + val logger; +}; + +std::unique_ptr emval_test_return_unique_ptr_lifetime(val logger) { + return std::unique_ptr(new UniquePtrLifetimeMock(logger)); +} + std::shared_ptr emval_test_return_shared_ptr() { return std::shared_ptr(new ValHolder(val::object())); } @@ -1957,6 +1974,9 @@ EMSCRIPTEN_BINDINGS(tests) { function("emval_test_return_unique_ptr", &emval_test_return_unique_ptr); + class_("UniquePtrLifetimeMock"); + function("emval_test_return_unique_ptr_lifetime", &emval_test_return_unique_ptr_lifetime); + function("emval_test_return_shared_ptr", &emval_test_return_shared_ptr); function("emval_test_return_empty_shared_ptr", &emval_test_return_empty_shared_ptr); function("emval_test_is_shared_ptr_null", &emval_test_is_shared_ptr_null); From 17278ec9c94c7da8721caf84abe6431da449425e Mon Sep 17 00:00:00 2001 From: Peter van der Tak Date: Fri, 7 Nov 2014 15:51:18 +0100 Subject: [PATCH 057/161] Return unique_ptr by releasing the raw pointer and returning that instead. --- system/include/emscripten/wire.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/system/include/emscripten/wire.h b/system/include/emscripten/wire.h index 94ce9690e1093..6c372d4d5f630 100644 --- a/system/include/emscripten/wire.h +++ b/system/include/emscripten/wire.h @@ -340,14 +340,14 @@ namespace emscripten { template struct GenericBindingType> { - typedef typename BindingType::WireType WireType; + typedef typename BindingType::WireType WireType; static WireType toWireType(std::unique_ptr p) { - return BindingType::toWireType(*p); + return BindingType::toWireType(p.release()); } static std::unique_ptr fromWireType(WireType wt) { - return std::unique_ptr(new T(std::move(BindingType::fromWireType(wt)))); + return std::unique_ptr(BindingType::fromWireType(wt)); } }; From 53aab28ef03cdb23eb97418d83e4189b00aaf224 Mon Sep 17 00:00:00 2001 From: Charlie Birks Date: Fri, 7 Nov 2014 15:07:56 +0000 Subject: [PATCH 058/161] Fullscreen api fixes --- src/library_html5.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/library_html5.js b/src/library_html5.js index 1a7cfc3ca56fc..24984a40b44ed 100644 --- a/src/library_html5.js +++ b/src/library_html5.js @@ -1415,7 +1415,7 @@ var LibraryJSEvents = { return _emscripten_do_request_fullscreen(target, strategy); }, - emscripten_request_fullscreen_strategy__deps: ['emscripten_do_request_fullscreen', '_currentFullscreenStrategy'], + emscripten_request_fullscreen_strategy__deps: ['emscripten_do_request_fullscreen', '_currentFullscreenStrategy', '_registerRestoreOldStyle'], emscripten_request_fullscreen_strategy: function(target, deferUntilInEventHandler, fullscreenStrategy) { var strategy = {}; strategy.scaleMode = {{{ makeGetValue('fullscreenStrategy', C_STRUCTS.EmscriptenFullscreenStrategy.scaleMode, 'i32') }}}; @@ -1480,6 +1480,7 @@ var LibraryJSEvents = { return {{{ cDefine('EMSCRIPTEN_RESULT_SUCCESS') }}}; }, + emscripten_exit_fullscreen__deps: ['_currentFullscreenStrategy'], emscripten_exit_fullscreen: function() { if (typeof JSEvents.fullscreenEnabled() === 'undefined') return {{{ cDefine('EMSCRIPTEN_RESULT_NOT_SUPPORTED') }}}; // Make sure no queued up calls will fire after this. @@ -1497,8 +1498,8 @@ var LibraryJSEvents = { return {{{ cDefine('EMSCRIPTEN_RESULT_NOT_SUPPORTED') }}}; } - if (strategy.canvasResizedCallback) { - Runtime.dynCall('iiii', strategy.canvasResizedCallback, [{{{ cDefine('EMSCRIPTEN_EVENT_CANVASRESIZED') }}}, 0, strategy.canvasResizedCallbackUserData]); + if (__currentFullscreenStrategy.canvasResizedCallback) { + Runtime.dynCall('iiii', __currentFullscreenStrategy.canvasResizedCallback, [{{{ cDefine('EMSCRIPTEN_EVENT_CANVASRESIZED') }}}, 0, __currentFullscreenStrategy.canvasResizedCallbackUserData]); } return {{{ cDefine('EMSCRIPTEN_RESULT_SUCCESS') }}}; From 1f089df891f09581fe89259b7e9efa0bd3b2085b Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Wed, 5 Nov 2014 11:18:26 -0800 Subject: [PATCH 059/161] Add tests for xmmintrin.h unaligned load and store --- tests/core/test_simd8.in | 17 +++++++++++++++++ tests/core/test_simd8.out | 4 ++++ tests/test_core.py | 11 +++++++++++ 3 files changed, 32 insertions(+) create mode 100644 tests/core/test_simd8.in create mode 100644 tests/core/test_simd8.out diff --git a/tests/core/test_simd8.in b/tests/core/test_simd8.in new file mode 100644 index 0000000000000..8358495cfaa59 --- /dev/null +++ b/tests/core/test_simd8.in @@ -0,0 +1,17 @@ +#include +#include + +char buffer[21]; + +int main() { + _mm_storeu_ps((float*)&buffer[1], (__m128){0.1, 2.3, 4.5, 6.7}); + __m128 y = _mm_loadu_ps((float*)&buffer[5]); + + float __attribute__((__aligned__(16))) ar[4]; + *(__m128 *)&ar = y; + for (int i = 0; i < 4; i++) { + printf("%f\n", ar[i]); + } + + return 0; +} diff --git a/tests/core/test_simd8.out b/tests/core/test_simd8.out new file mode 100644 index 0000000000000..7d833d0970ec1 --- /dev/null +++ b/tests/core/test_simd8.out @@ -0,0 +1,4 @@ +2.300000 +4.500000 +6.700000 +0.000000 diff --git a/tests/test_core.py b/tests/test_core.py index b94a10a00f984..d4693cb37575b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -5233,6 +5233,17 @@ def test_simd7(self): self.do_run_from_file(src, output) + def test_simd8(self): + # test_simd8 is to test unaligned load and store + Settings.PRECISE_F32 = 1 # SIMD currently requires Math.fround + if Settings.ASM_JS: Settings.ASM_JS = 2 # does not validate + if os.environ.get('EMCC_FAST_COMPILER') == '0': return self.skip('needs fastcomp') + + test_path = path_from_root('tests', 'core', 'test_simd8') + src, output = (test_path + s for s in ('.in', '.out')) + + self.do_run_from_file(src, output) + def test_simd_dyncall(self): if Settings.ASM_JS: Settings.ASM_JS = 2 # does not validate Settings.PRECISE_F32 = 1 # SIMD currently requires Math.fround From 41820d41f5ae9d2162ba2313e9058735b40bb6a3 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Wed, 5 Nov 2014 11:18:50 -0800 Subject: [PATCH 060/161] Update to latest development upstream. --- src/ecmascript_simd.js | 830 +++++++++++++++++++++++++---------------- 1 file changed, 499 insertions(+), 331 deletions(-) diff --git a/src/ecmascript_simd.js b/src/ecmascript_simd.js index acfaf2942d40b..3b2737ef4cb65 100644 --- a/src/ecmascript_simd.js +++ b/src/ecmascript_simd.js @@ -58,8 +58,19 @@ _SIMD_PRIVATE.maxNum = function(x, y) { // Type checking functions. -_SIMD_PRIVATE.isArrayBuffer = function(o) { - return (o instanceof ArrayBuffer); +_SIMD_PRIVATE.isTypedArray = function(o) { + return (o instanceof Int8Array) || + (o instanceof Uint8Array) || + (o instanceof Uint8ClampedArray) || + (o instanceof Int16Array) || + (o instanceof Uint16Array) || + (o instanceof Int32Array) || + (o instanceof Uint32Array) || + (o instanceof Float32Array) || + (o instanceof Float64Array) || + (o instanceof Int32x4Array) || + (o instanceof Float32x4Array) || + (o instanceof Float64x2Array); } _SIMD_PRIVATE.isNumber = function(o) { @@ -1229,21 +1240,20 @@ if (typeof SIMD.float32x4.sqrt === "undefined") { if (typeof SIMD.float32x4.swizzle === "undefined") { /** * @param {float32x4} t An instance of float32x4 to be swizzled. - * @param {integer} mask One of the 256 swizzle masks, for example, SIMD.XXXX. + * @param {integer} x - Index for lane x + * @param {integer} y - Index for lane y + * @param {integer} z - Index for lane z + * @param {integer} w - Index for lane w * @return {float32x4} New instance of float32x4 with lanes swizzled. */ - SIMD.float32x4.swizzle = function(t, mask) { + SIMD.float32x4.swizzle = function(t, x, y, z, w) { t = SIMD.float32x4(t); - var _x = (mask) & 0x3; - var _y = (mask >> 2) & 0x3; - var _z = (mask >> 4) & 0x3; - var _w = (mask >> 6) & 0x3; _SIMD_PRIVATE._f32x4[0] = t.x; _SIMD_PRIVATE._f32x4[1] = t.y; _SIMD_PRIVATE._f32x4[2] = t.z; _SIMD_PRIVATE._f32x4[3] = t.w; var storage = _SIMD_PRIVATE._f32x4; - return SIMD.float32x4(storage[_x], storage[_y], storage[_z], storage[_w]); + return SIMD.float32x4(storage[x], storage[y], storage[z], storage[w]); } } @@ -1514,177 +1524,231 @@ if (typeof SIMD.float32x4.not === "undefined") { if (typeof SIMD.float32x4.load === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @return {float32x4} New instance of float32x4. */ - SIMD.float32x4.load = function(buffer, byteOffset) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.float32x4.load = function(tarray, index) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 16) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); - var i8a = new Int8Array(buffer, byteOffset, 16); - for (var i = 0; i < 16; i++) - _SIMD_PRIVATE._i8x16[i] = i8a[i]; - return SIMD.float32x4(_SIMD_PRIVATE._f32x4[0], _SIMD_PRIVATE._f32x4[1], - _SIMD_PRIVATE._f32x4[2], _SIMD_PRIVATE._f32x4[3]); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 16) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); + var f32temp = _SIMD_PRIVATE._f32x4; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? f32temp : _SIMD_PRIVATE._i32x4) : + _SIMD_PRIVATE._f64x2; + var n = 16 / bpe; + for (var i = 0; i < n; ++i) + array[i] = tarray[index + i]; + return SIMD.float32x4(f32temp[0], f32temp[1], f32temp[2], f32temp[3]); } } if (typeof SIMD.float32x4.loadX === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @return {float32x4} New instance of float32x4. */ - SIMD.float32x4.loadX = function(buffer, byteOffset) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.float32x4.loadX = function(tarray, index) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 4) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); - var i8a = new Int8Array(buffer, byteOffset, 4); - for (var i = 0; i < 4; i++) - _SIMD_PRIVATE._i8x16[i] = i8a[i]; - return SIMD.float32x4(_SIMD_PRIVATE._f32x4[0], 0.0, 0.0, 0.0); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 4) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); + var f32temp = _SIMD_PRIVATE._f32x4; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? f32temp : _SIMD_PRIVATE._i32x4) : + _SIMD_PRIVATE._f64x2; + var n = 4 / bpe; + for (var i = 0; i < n; ++i) + array[i] = tarray[index + i]; + return SIMD.float32x4(f32temp[0], 0.0, 0.0, 0.0); } } if (typeof SIMD.float32x4.loadXY === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @return {float32x4} New instance of float32x4. */ - SIMD.float32x4.loadXY = function(buffer, byteOffset) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.float32x4.loadXY = function(tarray, index) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 8) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); - var i8a = new Int8Array(buffer, byteOffset, 8); - for (var i = 0; i < 8; i++) - _SIMD_PRIVATE._i8x16[i] = i8a[i]; - return SIMD.float32x4(_SIMD_PRIVATE._f32x4[0], _SIMD_PRIVATE._f32x4[1], 0.0, 0.0); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 8) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); + var f32temp = _SIMD_PRIVATE._f32x4; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? f32temp : _SIMD_PRIVATE._i32x4) : + _SIMD_PRIVATE._f64x2; + var n = 8 / bpe; + for (var i = 0; i < n; ++i) + array[i] = tarray[index + i]; + return SIMD.float32x4(f32temp[0], f32temp[1], 0.0, 0.0); } } if (typeof SIMD.float32x4.loadXYZ === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @return {float32x4} New instance of float32x4. */ - SIMD.float32x4.loadXYZ = function(buffer, byteOffset) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.float32x4.loadXYZ = function(tarray, index) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 12) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); - var i8a = new Int8Array(buffer, byteOffset, 12); - for (var i = 0; i < 12; i++) - _SIMD_PRIVATE._i8x16[i] = i8a[i]; - return SIMD.float32x4(_SIMD_PRIVATE._f32x4[0], _SIMD_PRIVATE._f32x4[1], - _SIMD_PRIVATE._f32x4[2], 0.0); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 12) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); + var f32temp = _SIMD_PRIVATE._f32x4; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? f32temp : _SIMD_PRIVATE._i32x4) : + _SIMD_PRIVATE._f64x2; + var n = 12 / bpe; + for (var i = 0; i < n; ++i) + array[i] = tarray[index + i]; + return SIMD.float32x4(f32temp[0], f32temp[1], f32temp[2], 0.0); } } if (typeof SIMD.float32x4.store === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @param {float32x4} value An instance of float32x4. * @return {void} */ - SIMD.float32x4.store = function(buffer, byteOffset, value) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.float32x4.store = function(tarray, index, value) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 16) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 16) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); value = SIMD.float32x4(value); _SIMD_PRIVATE._f32x4[0] = value.x; _SIMD_PRIVATE._f32x4[1] = value.y; _SIMD_PRIVATE._f32x4[2] = value.z; _SIMD_PRIVATE._f32x4[3] = value.w; - var i8a = new Int8Array(buffer, byteOffset, 16); - for (var i = 0; i < 16; i++) - i8a[i] = _SIMD_PRIVATE._i8x16[i]; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4) : + _SIMD_PRIVATE._f64x2; + var n = 16 / bpe; + for (var i = 0; i < n; ++i) + tarray[index + i] = array[i]; } } if (typeof SIMD.float32x4.storeX === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @param {float32x4} value An instance of float32x4. * @return {void} */ - SIMD.float32x4.storeX = function(buffer, byteOffset, value) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.float32x4.storeX = function(tarray, index, value) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 4) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 4) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); value = SIMD.float32x4(value); - _SIMD_PRIVATE._f32x4[0] = value.x; - var i8a = new Int8Array(buffer, byteOffset, 4); - for (var i = 0; i < 4; i++) - i8a[i] = _SIMD_PRIVATE._i8x16[i]; + if (bpe == 8) { + // tarray's elements are too wide. Just create a new view; this is rare. + var view = new Float32Array(tarray.buffer, tarray.byteOffset + index * 8, 1); + view[0] = value.x; + } else { + _SIMD_PRIVATE._f32x4[0] = value.x; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4); + var n = 4 / bpe; + for (var i = 0; i < n; ++i) + tarray[index + i] = array[i]; + } } } if (typeof SIMD.float32x4.storeXY === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @param {float32x4} value An instance of float32x4. * @return {void} */ - SIMD.float32x4.storeXY = function(buffer, byteOffset, value) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.float32x4.storeXY = function(tarray, index, value) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 8) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 8) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); value = SIMD.float32x4(value); _SIMD_PRIVATE._f32x4[0] = value.x; _SIMD_PRIVATE._f32x4[1] = value.y; - var i8a = new Int8Array(buffer, byteOffset, 8); - for (var i = 0; i < 8; i++) - i8a[i] = _SIMD_PRIVATE._i8x16[i]; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4) : + _SIMD_PRIVATE._f64x2; + var n = 8 / bpe; + for (var i = 0; i < n; ++i) + tarray[index + i] = array[i]; } } if (typeof SIMD.float32x4.storeXYZ === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @param {float32x4} value An instance of float32x4. * @return {void} */ - SIMD.float32x4.storeXYZ = function(buffer, byteOffset, value) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.float32x4.storeXYZ = function(tarray, index, value) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 12) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 12) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); value = SIMD.float32x4(value); - _SIMD_PRIVATE._f32x4[0] = value.x; - _SIMD_PRIVATE._f32x4[1] = value.y; - _SIMD_PRIVATE._f32x4[2] = value.z; - var i8a = new Int8Array(buffer, byteOffset, 12); - for (var i = 0; i < 12; i++) - i8a[i] = _SIMD_PRIVATE._i8x16[i]; + if (bpe == 8) { + // tarray's elements are too wide. Just create a new view; this is rare. + var view = new Float32Array(tarray.buffer, tarray.byteOffset + index * 8, 3); + view[0] = value.x; + view[1] = value.y; + view[2] = value.z; + } else { + _SIMD_PRIVATE._f32x4[0] = value.x; + _SIMD_PRIVATE._f32x4[1] = value.y; + _SIMD_PRIVATE._f32x4[2] = value.z; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4); + var n = 12 / bpe; + for (var i = 0; i < n; ++i) + tarray[index + i] = array[i]; + } } } @@ -1874,17 +1938,16 @@ if (typeof SIMD.float64x2.sqrt === "undefined") { if (typeof SIMD.float64x2.swizzle === "undefined") { /** * @param {float64x2} t An instance of float64x2 to be swizzled. - * @param {integer} mask One of the 4 swizzle masks, for example, SIMD.XY. + * @param {integer} x - Index for lane x + * @param {integer} y - Index for lane y * @return {float64x2} New instance of float64x2 with lanes swizzled. */ - SIMD.float64x2.swizzle = function(t, mask) { + SIMD.float64x2.swizzle = function(t, x, y) { t = SIMD.float64x2(t); - var _x = (mask) & 0x1; - var _y = (mask >> 1) & 0x1; var storage = _SIMD_PRIVATE._f64x2; storage[0] = t.x; storage[1] = t.y; - return SIMD.float64x2(storage[_x], storage[_y]); + return SIMD.float64x2(storage[x], storage[y]); } } @@ -2056,86 +2119,108 @@ if (typeof SIMD.float64x2.select === "undefined") { if (typeof SIMD.float64x2.load === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @return {float64x2} New instance of float64x2. */ - SIMD.float64x2.load = function(buffer, byteOffset) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.float64x2.load = function(tarray, index) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 16) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); - var i8a = new Int8Array(buffer, byteOffset, 16); - for (var i = 0; i < 16; i++) - _SIMD_PRIVATE._i8x16[i] = i8a[i]; - return SIMD.float64x2(_SIMD_PRIVATE._f64x2[0], _SIMD_PRIVATE._f64x2[1]); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 16) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); + var f64temp = _SIMD_PRIVATE._f64x2; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4) : + f64temp; + var n = 16 / bpe; + for (var i = 0; i < n; ++i) + array[i] = tarray[index + i]; + return SIMD.float64x2(f64temp[0], f64temp[1]); } } if (typeof SIMD.float64x2.loadX === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @return {float64x2} New instance of float64x2. */ - SIMD.float64x2.loadX = function(buffer, byteOffset) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.float64x2.loadX = function(tarray, index) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 8) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); - var i8a = new Int8Array(buffer, byteOffset, 8); - for (var i = 0; i < 8; i++) - _SIMD_PRIVATE._i8x16[i] = i8a[i]; - return SIMD.float64x2(_SIMD_PRIVATE._f64x2[0], 0.0); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 8) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); + var f64temp = _SIMD_PRIVATE._f64x2; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4) : + f64temp; + var n = 8 / bpe; + for (var i = 0; i < n; ++i) + array[i] = tarray[index + i]; + return SIMD.float64x2(f64temp[0], 0.0); } } if (typeof SIMD.float64x2.store === "undefined") { /** - * @param {ArrayBuffer} f64a An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @param {float64x2} value An instance of float64x2. * @return {void} */ - SIMD.float64x2.store = function(buffer, byteOffset, value) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.float64x2.store = function(tarray, index, value) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 16) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 16) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); value = SIMD.float64x2(value); _SIMD_PRIVATE._f64x2[0] = value.x; _SIMD_PRIVATE._f64x2[1] = value.y; - var i8a = new Int8Array(buffer, byteOffset, 16); - for (var i = 0; i < 16; i++) - i8a[i] = _SIMD_PRIVATE._i8x16[i]; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4) : + _SIMD_PRIVATE._f64x2; + var n = 16 / bpe; + for (var i = 0; i < n; ++i) + tarray[index + i] = array[i]; } } if (typeof SIMD.float64x2.storeX === "undefined") { /** - * @param {ArrayBuffer} f64a An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @param {float64x2} value An instance of float64x2. * @return {void} */ - SIMD.float64x2.storeX = function(buffer, byteOffset, value) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.float64x2.storeX = function(tarray, index, value) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 8) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 8) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); value = SIMD.float64x2(value); _SIMD_PRIVATE._f64x2[0] = value.x; - var i8a = new Int8Array(buffer, byteOffset, 8); - for (var i = 0; i < 8; i++) - i8a[i] = _SIMD_PRIVATE._i8x16[i]; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4) : + _SIMD_PRIVATE._f64x2; + var n = 8 / bpe; + for (var i = 0; i < n; ++i) + tarray[index + i] = array[i]; } } @@ -2243,21 +2328,20 @@ if (typeof SIMD.int32x4.mul === "undefined") { if (typeof SIMD.int32x4.swizzle === "undefined") { /** * @param {int32x4} t An instance of float32x4 to be swizzled. - * @param {integer} mask One of the 256 swizzle masks, for example, SIMD.XXXX. + * @param {integer} x - Index for lane x + * @param {integer} y - Index for lane y + * @param {integer} z - Index for lane z + * @param {integer} w - Index for lane w * @return {int32x4} New instance of float32x4 with lanes swizzled. */ - SIMD.int32x4.swizzle = function(t, mask) { + SIMD.int32x4.swizzle = function(t, x, y, z, w) { t = SIMD.int32x4(t); - var _x = (mask) & 0x3; - var _y = (mask >> 2) & 0x3; - var _z = (mask >> 4) & 0x3; - var _w = (mask >> 6) & 0x3; var storage = _SIMD_PRIVATE._i32x4; storage[0] = t.x; storage[1] = t.y; storage[2] = t.z; storage[3] = t.w; - return SIMD.int32x4(storage[_x], storage[_y], storage[_z], storage[_w]); + return SIMD.int32x4(storage[x], storage[y], storage[z], storage[w]); } } @@ -2607,7 +2691,7 @@ if (typeof SIMD.int32x4.shiftLeftByScalar === "undefined") { SIMD.int32x4.shiftLeftByScalar = function(a, bits) { a = SIMD.int32x4(a); if (bits>>>0 >= 32) - return SIMD.int32x4.zero(); + return SIMD.int32x4.splat(0.0); var x = a.x << bits; var y = a.y << bits; var z = a.z << bits; @@ -2625,7 +2709,7 @@ if (typeof SIMD.int32x4.shiftRightLogicalByScalar === "undefined") { SIMD.int32x4.shiftRightLogicalByScalar = function(a, bits) { a = SIMD.int32x4(a); if (bits>>>0 >= 32) - return SIMD.int32x4.zero(); + return SIMD.int32x4.splat(0.0); var x = a.x >>> bits; var y = a.y >>> bits; var z = a.z >>> bits; @@ -2708,177 +2792,231 @@ if (typeof SIMD.int32x4.shiftRightArithmetic === "undefined") { if (typeof SIMD.int32x4.load === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @return {int32x4} New instance of int32x4. */ - SIMD.int32x4.load = function(buffer, byteOffset) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.int32x4.load = function(tarray, index) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 16) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); - var i8a = new Int8Array(buffer, byteOffset, 16); - for (var i = 0; i < 16; i++) - _SIMD_PRIVATE._i8x16[i] = i8a[i]; - return SIMD.int32x4(_SIMD_PRIVATE._i32x4[0], _SIMD_PRIVATE._i32x4[1], - _SIMD_PRIVATE._i32x4[2], _SIMD_PRIVATE._i32x4[3]); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 16) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); + var i32temp = _SIMD_PRIVATE._i32x4; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : i32temp) : + _SIMD_PRIVATE._f64x2; + var n = 16 / bpe; + for (var i = 0; i < n; ++i) + array[i] = tarray[index + i]; + return SIMD.int32x4(i32temp[0], i32temp[1], i32temp[2], i32temp[3]); } } if (typeof SIMD.int32x4.loadX === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @return {int32x4} New instance of int32x4. */ - SIMD.int32x4.loadX = function(buffer, byteOffset) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.int32x4.loadX = function(tarray, index) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 4) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); - var i8a = new Int8Array(buffer, byteOffset, 4); - for (var i = 0; i < 4; i++) - _SIMD_PRIVATE._i8x16[i] = i8a[i]; - return SIMD.int32x4(_SIMD_PRIVATE._i32x4[0], 0, 0, 0); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 4) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); + var i32temp = _SIMD_PRIVATE._i32x4; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : i32temp) : + _SIMD_PRIVATE._f64x2; + var n = 4 / bpe; + for (var i = 0; i < n; ++i) + array[i] = tarray[index + i]; + return SIMD.int32x4(i32temp[0], 0, 0, 0); } } if (typeof SIMD.int32x4.loadXY === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @return {int32x4} New instance of int32x4. */ - SIMD.int32x4.loadXY = function(buffer, byteOffset) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.int32x4.loadXY = function(tarray, index) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 8) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); - var i8a = new Int8Array(buffer, byteOffset, 8); - for (var i = 0; i < 8; i++) - _SIMD_PRIVATE._i8x16[i] = i8a[i]; - return SIMD.int32x4(_SIMD_PRIVATE._i32x4[0], _SIMD_PRIVATE._i32x4[1], 0, 0); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 8) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); + var i32temp = _SIMD_PRIVATE._i32x4; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : i32temp) : + _SIMD_PRIVATE._f64x2; + var n = 8 / bpe; + for (var i = 0; i < n; ++i) + array[i] = tarray[index + i]; + return SIMD.int32x4(i32temp[0], i32temp[1], 0, 0); } } if (typeof SIMD.int32x4.loadXYZ === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @return {int32x4} New instance of int32x4. */ - SIMD.int32x4.loadXYZ = function(buffer, byteOffset) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.int32x4.loadXYZ = function(tarray, index) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 12) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); - var i8a = new Int8Array(buffer, byteOffset, 12); - for (var i = 0; i < 12; i++) - _SIMD_PRIVATE._i8x16[i] = i8a[i]; - return SIMD.int32x4(_SIMD_PRIVATE._i32x4[0], _SIMD_PRIVATE._i32x4[1], - _SIMD_PRIVATE._i32x4[2], 0); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 12) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); + var i32temp = _SIMD_PRIVATE._i32x4; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : i32temp) : + _SIMD_PRIVATE._f64x2; + var n = 12 / bpe; + for (var i = 0; i < n; ++i) + array[i] = tarray[index + i]; + return SIMD.int32x4(i32temp[0], i32temp[1], i32temp[2], 0); } } if (typeof SIMD.int32x4.store === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @param {int32x4} value An instance of int32x4. * @return {void} */ - SIMD.int32x4.store = function(buffer, byteOffset, value) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.int32x4.store = function(tarray, index, value) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 16) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 16) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); value = SIMD.int32x4(value); _SIMD_PRIVATE._i32x4[0] = value.x; _SIMD_PRIVATE._i32x4[1] = value.y; _SIMD_PRIVATE._i32x4[2] = value.z; _SIMD_PRIVATE._i32x4[3] = value.w; - var i8a = new Int8Array(buffer, byteOffset, 16); - for (var i = 0; i < 16; i++) - i8a[i] = _SIMD_PRIVATE._i8x16[i]; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4) : + _SIMD_PRIVATE._f64x2; + var n = 16 / bpe; + for (var i = 0; i < n; ++i) + tarray[index + i] = array[i]; } } if (typeof SIMD.int32x4.storeX === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @param {int32x4} value An instance of int32x4. * @return {void} */ - SIMD.int32x4.storeX = function(buffer, byteOffset, value) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.int32x4.storeX = function(tarray, index, value) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 4) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 4) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); value = SIMD.int32x4(value); - _SIMD_PRIVATE._i32x4[0] = value.x; - var i8a = new Int8Array(buffer, byteOffset, 4); - for (var i = 0; i < 4; i++) - i8a[i] = _SIMD_PRIVATE._i8x16[i]; + if (bpe == 8) { + // tarray's elements are too wide. Just create a new view; this is rare. + var view = new Int32Array(tarray.buffer, tarray.byteOffset + index * 8, 1); + view[0] = value.x; + } else { + _SIMD_PRIVATE._i32x4[0] = value.x; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4); + var n = 4 / bpe; + for (var i = 0; i < n; ++i) + tarray[index + i] = array[i]; + } } } if (typeof SIMD.int32x4.storeXY === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @param {int32x4} value An instance of int32x4. * @return {void} */ - SIMD.int32x4.storeXY = function(buffer, byteOffset, value) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.int32x4.storeXY = function(tarray, index, value) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 8) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 8) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); value = SIMD.int32x4(value); _SIMD_PRIVATE._i32x4[0] = value.x; _SIMD_PRIVATE._i32x4[1] = value.y; - var i8a = new Int8Array(buffer, byteOffset, 8); - for (var i = 0; i < 8; i++) - i8a[i] = _SIMD_PRIVATE._i8x16[i]; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4) : + _SIMD_PRIVATE._f64x2; + var n = 8 / bpe; + for (var i = 0; i < n; ++i) + tarray[index + i] = array[i]; } } if (typeof SIMD.int32x4.storeXYZ === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @param {int32x4} value An instance of int32x4. * @return {void} */ - SIMD.int32x4.storeXYZ = function(buffer, byteOffset, value) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.int32x4.storeXYZ = function(tarray, index, value) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 12) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 12) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); value = SIMD.int32x4(value); - _SIMD_PRIVATE._i32x4[0] = value.x; - _SIMD_PRIVATE._i32x4[1] = value.y; - _SIMD_PRIVATE._i32x4[2] = value.z; - var i8a = new Int8Array(buffer, byteOffset, 12); - for (var i = 0; i < 12; i++) - i8a[i] = _SIMD_PRIVATE._i8x16[i]; + if (bpe == 8) { + // tarray's elements are too wide. Just create a new view; this is rare. + var view = new Int32Array(tarray.buffer, tarray.byteOffset + index * 8, 3); + view[0] = value.x; + view[1] = value.y; + view[2] = value.z; + } else { + _SIMD_PRIVATE._i32x4[0] = value.x; + _SIMD_PRIVATE._i32x4[1] = value.y; + _SIMD_PRIVATE._i32x4[2] = value.z; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4); + var n = 12 / bpe; + for (var i = 0; i < n; ++i) + tarray[index + i] = array[i]; + } } } @@ -3366,47 +3504,62 @@ if (typeof SIMD.int16x8.shiftRightArithmetic === "undefined") { if (typeof SIMD.int16x8.load === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @return {int16x8} New instance of int16x8. */ - SIMD.int16x8.load = function(buffer, byteOffset) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.int16x8.load = function(tarray, index) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 16) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); - var i16a = new Int16Array(buffer, byteOffset, 8); - return SIMD.int16x8(i16a[0], i16a[1], i16a[2], i16a[3], - i16a[4], i16a[5], i16a[6], i16a[7]); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 16) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); + var i16temp = _SIMD_PRIVATE._i16x8; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? i16temp : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4) : + _SIMD_PRIVATE._f64x2; + var n = 16 / bpe; + for (var i = 0; i < n; ++i) + array[i] = tarray[index + i]; + return SIMD.int16x8(i16temp[0], i16temp[1], i16temp[2], i16temp[3], + i16temp[4], i16temp[5], i16temp[6], i16temp[7]); } } if (typeof SIMD.int16x8.store === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @param {int16x8} value An instance of int16x8. * @return {void} */ - SIMD.int16x8.store = function(buffer, byteOffset, value) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.int16x8.store = function(tarray, index, value) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 16) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 16) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); value = SIMD.int16x8(value); - var i16a = new Int16Array(buffer, byteOffset, 8); - i16a[0] = value.s0; - i16a[1] = value.s1; - i16a[2] = value.s2; - i16a[3] = value.s3; - i16a[4] = value.s4; - i16a[5] = value.s5; - i16a[6] = value.s6; - i16a[7] = value.s7; + _SIMD_PRIVATE._i16x8[0] = value.s0; + _SIMD_PRIVATE._i16x8[1] = value.s1; + _SIMD_PRIVATE._i16x8[2] = value.s2; + _SIMD_PRIVATE._i16x8[3] = value.s3; + _SIMD_PRIVATE._i16x8[4] = value.s4; + _SIMD_PRIVATE._i16x8[5] = value.s5; + _SIMD_PRIVATE._i16x8[6] = value.s6; + _SIMD_PRIVATE._i16x8[7] = value.s7; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4) : + _SIMD_PRIVATE._f64x2; + var n = 16 / bpe; + for (var i = 0; i < n; ++i) + tarray[index + i] = array[i]; } } @@ -4140,57 +4293,72 @@ if (typeof SIMD.int8x16.shiftRightArithmetic === "undefined") { if (typeof SIMD.int8x16.load === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @return {int8x16} New instance of int8x16. */ - SIMD.int8x16.load = function(buffer, byteOffset) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.int8x16.load = function(tarray, index) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 16) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); - var i8a = new Int8Array(buffer, byteOffset, 16); - return SIMD.int8x16(i8a[0], i8a[1], i8a[2], i8a[3], - i8a[4], i8a[5], i8a[6], i8a[7], - i8a[8], i8a[9], i8a[10], i8a[11], - i8a[12], i8a[13], i8a[14], i8a[15]); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 16) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); + var i8temp = _SIMD_PRIVATE._i16x8; + var array = bpe == 1 ? i8temp : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4) : + _SIMD_PRIVATE._f64x2; + var n = 16 / bpe; + for (var i = 0; i < n; ++i) + array[i] = tarray[index + i]; + return SIMD.int8x16(i8temp[0], i8temp[1], i8temp[2], i8temp[3], + i8temp[4], i8temp[5], i8temp[6], i8temp[7], + i8temp[8], i8temp[9], i8temp[10], i8temp[11], + i8temp[12], i8temp[13], i8temp[14], i8temp[15]); } } if (typeof SIMD.int8x16.store === "undefined") { /** - * @param {ArrayBuffer} buffer An instance of ArrayBuffer. - * @param {Number} byteOffset An instance of Number. + * @param {Typed array} tarray An instance of a typed array. + * @param {Number} index An instance of Number. * @param {int8x16} value An instance of int8x16. * @return {void} */ - SIMD.int8x16.store = function(buffer, byteOffset, value) { - if (!_SIMD_PRIVATE.isArrayBuffer(buffer)) - throw new TypeError("The 1st argument must be an ArrayBuffer."); - if (!_SIMD_PRIVATE.isNumber(byteOffset)) + SIMD.int8x16.store = function(tarray, index, value) { + if (!_SIMD_PRIVATE.isTypedArray(tarray)) + throw new TypeError("The 1st argument must be a typed array."); + if (!_SIMD_PRIVATE.isNumber(index)) throw new TypeError("The 2nd argument must be a Number."); - if (byteOffset < 0 || (byteOffset + 16) > buffer.byteLength) - throw new RangeError("The value of byteOffset is invalid."); + var bpe = tarray.BYTES_PER_ELEMENT; + if (index < 0 || (index * bpe + 16) > tarray.byteLength) + throw new RangeError("The value of index is invalid."); value = SIMD.int8x16(value); - var i8a = new Int8Array(buffer, byteOffset, 16); - i8a[0] = value.s0; - i8a[1] = value.s1; - i8a[2] = value.s2; - i8a[3] = value.s3; - i8a[4] = value.s4; - i8a[5] = value.s5; - i8a[6] = value.s6; - i8a[7] = value.s7; - i8a[8] = value.s8; - i8a[9] = value.s9; - i8a[10] = value.s10; - i8a[11] = value.s11; - i8a[12] = value.s12; - i8a[13] = value.s13; - i8a[14] = value.s14; - i8a[15] = value.s15; + _SIMD_PRIVATE._i8x16[0] = value.s0; + _SIMD_PRIVATE._i8x16[1] = value.s1; + _SIMD_PRIVATE._i8x16[2] = value.s2; + _SIMD_PRIVATE._i8x16[3] = value.s3; + _SIMD_PRIVATE._i8x16[4] = value.s4; + _SIMD_PRIVATE._i8x16[5] = value.s5; + _SIMD_PRIVATE._i8x16[6] = value.s6; + _SIMD_PRIVATE._i8x16[7] = value.s7; + _SIMD_PRIVATE._i8x16[8] = value.s8; + _SIMD_PRIVATE._i8x16[9] = value.s9; + _SIMD_PRIVATE._i8x16[10] = value.s10; + _SIMD_PRIVATE._i8x16[11] = value.s11; + _SIMD_PRIVATE._i8x16[12] = value.s12; + _SIMD_PRIVATE._i8x16[13] = value.s13; + _SIMD_PRIVATE._i8x16[14] = value.s14; + _SIMD_PRIVATE._i8x16[15] = value.s15; + var array = bpe == 1 ? _SIMD_PRIVATE._i8x16 : + bpe == 2 ? _SIMD_PRIVATE._i16x8 : + bpe == 4 ? (tarray instanceof Float32Array ? _SIMD_PRIVATE._f32x4 : _SIMD_PRIVATE._i32x4) : + _SIMD_PRIVATE._f64x2; + var n = 16 / bpe; + for (var i = 0; i < n; ++i) + tarray[index + i] = array[i]; } } From b5f2f384417f62b988c8af4e0b54103e0234e37d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 7 Nov 2014 23:14:28 +0200 Subject: [PATCH 061/161] In proxy to worker mode, the proxy side GL object does not have a .canvas member because it can't access the DOM, so avoid accessing that object in proxy to worker mode. --- src/library_gl.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/library_gl.js b/src/library_gl.js index 65d5e796b95bc..c85c2c19f5ac9 100644 --- a/src/library_gl.js +++ b/src/library_gl.js @@ -817,7 +817,7 @@ var LibraryGL = { GLctx: ctx }; // Store the created context object so that we can access the context given a canvas without having to pass the parameters again. - ctx.canvas.GLctxObject = context; + if (ctx.canvas) ctx.canvas.GLctxObject = context; GL.contexts[handle] = context; if (typeof webGLContextAttributes['webGLContextAttributes'] === 'undefined' || webGLContextAttributes.enableExtensionsByDefault) { GL.initExtensions(context); @@ -839,7 +839,7 @@ var LibraryGL = { deleteContext: function(contextHandle) { if (GL.currentContext === GL.contexts[contextHandle]) GL.currentContext = 0; - if (GL.contexts[contextHandle]) GL.contexts[contextHandle].GLctx.canvas.GLctxObject = undefined; // Make sure the canvas object no longer refers to the context object so there are no GC surprises. + if (GL.contexts[contextHandle] && GL.contexts[contextHandle].GLctx.canvas) GL.contexts[contextHandle].GLctx.canvas.GLctxObject = undefined; // Make sure the canvas object no longer refers to the context object so there are no GC surprises. GL.contexts[contextHandle] = null; }, From 32c6ccec13055ebcf3a822b48610287c4c8b4b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 31 Oct 2014 19:37:01 -0400 Subject: [PATCH 062/161] When user deletes a WebGL context, disconnect all event handlers associated with that context so that phantom GL context loss events or similar are not passed to user from contexts that are already supposed to be freed. --- src/library_gl.js | 1 + src/library_html5.js | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/src/library_gl.js b/src/library_gl.js index c85c2c19f5ac9..798fec905908f 100644 --- a/src/library_gl.js +++ b/src/library_gl.js @@ -839,6 +839,7 @@ var LibraryGL = { deleteContext: function(contextHandle) { if (GL.currentContext === GL.contexts[contextHandle]) GL.currentContext = 0; + if (typeof JSEvents === 'object') JSEvents.removeAllHandlersOnTarget(GL.contexts[contextHandle].canvas); // Release all JS event handlers on the DOM element that the GL context is associated with since the context is now deleted. if (GL.contexts[contextHandle] && GL.contexts[contextHandle].GLctx.canvas) GL.contexts[contextHandle].GLctx.canvas.GLctxObject = undefined; // Make sure the canvas object no longer refers to the context object so there are no GC surprises. GL.contexts[contextHandle] = null; }, diff --git a/src/library_html5.js b/src/library_html5.js index 24984a40b44ed..57f37ae5aca60 100644 --- a/src/library_html5.js +++ b/src/library_html5.js @@ -125,6 +125,16 @@ var LibraryJSEvents = { isInternetExplorer: function() { return navigator.userAgent.indexOf('MSIE') !== -1 || navigator.appVersion.indexOf('Trident/') > 0; }, + // Removes all event handlers on the given DOM element of the given type. Pass in eventType == undefined/null to remove all event handlers regardless of the type. + removeAllHandlersOnTarget: function(target, eventTypeString) { + for(var i = 0; i < JSEvents.eventHandlers.length; ++i) { + if (JSEvents.eventHandlers[i].target == target && + (!eventType || eventTypeString == JSEvents.eventHandlers[i].eventTypeString)) { + JSEvents._removeHandler(i--); + } + } + }, + _removeHandler: function(i) { var h = JSEvents.eventHandlers[i]; h.target.removeEventListener(h.eventTypeString, h.eventListenerFunc, h.useCapture); From 55fead4b89ca80a3403c3d0d6ad278b6fe8e834b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Nov 2014 14:51:48 -0800 Subject: [PATCH 063/161] remove warning on exiting runtime; we assert on entering it, which is enough; fixes #2960 --- src/preamble.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index e992525bba8e9..11afecdfdcc99 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1181,11 +1181,6 @@ function preMain() { } function exitRuntime() { -#if ASSERTIONS - if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { - Module.printErr('Exiting runtime. Any attempt to access the compiled C code may fail from now. If you want to keep the runtime alive, set Module["noExitRuntime"] = true or build with -s NO_EXIT_RUNTIME=1'); - } -#endif callRuntimeCallbacks(__ATEXIT__); runtimeExited = true; } From 8fe33886de7ed6a4f0c4423eac370f27110d3855 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Sat, 8 Nov 2014 00:18:41 +0100 Subject: [PATCH 064/161] fix indentation --- src/preamble.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index 68a110cbea402..d16a8f99cc4eb 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -453,13 +453,13 @@ var cwrap, ccall; return eval(funcstr); }; #else - // NO_DYNAMIC_EXECUTION is on, so we can't use the fast version of cwrap. - // Fall back to returning a bound version of ccall. - cwrap = function cwrap(ident, returnType, argTypes) { - return function() { - return ccall(ident, returnType, argTypes, arguments); - } - } + // NO_DYNAMIC_EXECUTION is on, so we can't use the fast version of cwrap. + // Fall back to returning a bound version of ccall. + cwrap = function cwrap(ident, returnType, argTypes) { + return function() { + return ccall(ident, returnType, argTypes, arguments); + } + } #endif })(); Module["cwrap"] = cwrap; From 36881195bb31da800c3b181177cb8fa23beb62c1 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Sat, 8 Nov 2014 00:31:53 +0100 Subject: [PATCH 065/161] Add a warning when using slow cwrap When ASSERTIONS and NO_DYNAMIC_EXECUTION are set, and cwrap is used, display a warning. --- src/preamble.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index d16a8f99cc4eb..aa1af05e71b22 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -456,9 +456,13 @@ var cwrap, ccall; // NO_DYNAMIC_EXECUTION is on, so we can't use the fast version of cwrap. // Fall back to returning a bound version of ccall. cwrap = function cwrap(ident, returnType, argTypes) { - return function() { - return ccall(ident, returnType, argTypes, arguments); - } + return function() { +#if ASSERTIONS + Runtime.warnOnce('NO_DYNAMIC_EXECUTION was set, ' + + 'using slow cwrap implementation'); +#endif + return ccall(ident, returnType, argTypes, arguments); + } } #endif })(); From cbc5fc12632fc5b224b50b136ade89ff9e5cb0c9 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Nov 2014 17:32:00 -0800 Subject: [PATCH 066/161] never free Values, leave them all alive until process exit --- tools/optimizer/minijson.h | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 5974c28168338..5324deed2cc73 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -11,7 +11,6 @@ #include #include -#include #include #include #include @@ -27,15 +26,15 @@ struct Value; void dump(const char *str, Ref node, bool pretty=false); // Reference to a value. Simple shared_ptr, plus [] operator for convenience - we work on lots of arrays -class Ref : public std::shared_ptr { -public: - Ref() { - reset(); - } - Ref(Value *v) { - reset(v); - } +struct Ref { + Value* inst; + + Ref(Value *v=nullptr) : inst(v) {} + + Value* get() { return inst; } + Value& operator*() { return *inst; } + Value* operator->() { return inst; } Ref& operator[](unsigned x); // special conveniences From e61991b2a46a9e66afc1fdbb3340ab6cbf2cf4ab Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Nov 2014 17:42:17 -0800 Subject: [PATCH 067/161] use arena allocator --- tools/optimizer/minijson.h | 39 +++++++++++----- tools/optimizer/optimizer.cpp | 87 ++++++++++++++++++++--------------- 2 files changed, 78 insertions(+), 48 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 5324deed2cc73..f4a1ceb524cb4 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -45,7 +45,19 @@ struct Ref { bool operator!(); // check if null, in effect }; -typedef std::vector ArrayStorage; +// Arena allocation, free it all on process exit + +struct Arena { + #define CHUNK_SIZE 1000 + std::vector chunks; + int index; // in last chunk + + Arena() : index(0) {} + + Ref alloc(); +}; + +Arena arena; // Main value type struct Value { @@ -59,6 +71,8 @@ struct Value { Type type; + typedef std::vector ArrayStorage; + union { // TODO: optimize std::string *str; double num; @@ -233,7 +247,7 @@ struct Value { skip(); setArray(); while (*curr != ']') { - Ref temp(new Value); + Ref temp = arena.alloc(); arr->push_back(temp); curr = temp->parse(curr); skip(); @@ -324,13 +338,13 @@ struct Value { if (old != size) arr->resize(size); if (old < size) { for (unsigned i = old; i < size; i++) { - (*arr)[i] = new Value(); + (*arr)[i] = arena.alloc(); } } } Ref& operator[](unsigned x) { // tolerant, returns Null on out of bounds access. makes it convenient to check e.g. [3] on an if node - static Ref null = new Value(); // TODO: freeze this + static Ref null = arena.alloc(); // TODO: freeze this assert(isArray()); if (x >= arr->size()) return null; return (*arr)[x]; @@ -372,13 +386,6 @@ struct Value { // Bool operations }; -// Convenience class to construct arrays -struct ArrayValue : public Value { - ArrayValue() { - setArray(); - } -}; - // Ref methods Ref& Ref::operator[](unsigned x) { @@ -401,6 +408,16 @@ bool Ref::operator!() { return get()->isNull(); } +// Arena methods + +Ref Arena::alloc() { + if (chunks.size() == 0 || index == CHUNK_SIZE) { + chunks.push_back(new Value[CHUNK_SIZE]); + index = 0; + } + return &chunks.back()[index++]; +} + // dump void dump(const char *str, Ref node, bool pretty) { diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index c25b618008de2..e8a7694a74093 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -102,7 +102,7 @@ Ref getStatements(Ref node) { } else if (node[0] == "block") { return node[1]; } else { - return new Value(); + return arena.alloc(); } } @@ -123,6 +123,7 @@ AsmType detectType(Ref node, AsmData *asmData=nullptr, bool inVarDef=false); Ref makeEmpty(); bool isEmpty(Ref node); Ref makeAsmVarDef(const std::string& v_, AsmType type); +Ref makeArray(); Ref makeNum(double x); Ref makeName(const std::string& str); Ref makeAsmCoercion(Ref node, AsmType type); @@ -227,7 +228,7 @@ struct AsmData { } } // calculate variable definitions - Ref varDefs = new ArrayValue(); + Ref varDefs = makeArray(); for (auto v : vars) { varDefs->push_back(makeAsmVarDef(v->first, v->second.type)); } @@ -248,7 +249,7 @@ struct AsmData { int next = 0; for (auto param : func[2]->getArray()) { std::string& str = param->getString(); - stats[next++] = make1("stat", make3("assign", &(new Value())->setBool(true), makeName(str), makeAsmCoercion(makeName(str), locals[str].type))); + stats[next++] = make1("stat", make3("assign", &(arena.alloc())->setBool(true), makeName(str), makeAsmCoercion(makeName(str), locals[str].type))); } if (varDefs->size()) { stats[next] = make1("var", varDefs); @@ -388,31 +389,43 @@ AsmType detectType(Ref node, AsmData *asmData, bool inVarDef) { // Constructions TODO: share common constructions, and assert they remain frozen +Ref makeArray() { + return &arena.alloc()->setArray(); +} + +Ref makeString(const char *s) { + return &arena.alloc()->setString(s); +} + +Ref makeString(const std::string& s) { + return &arena.alloc()->setString(s); +} + Ref makeEmpty() { - Ref ret(new ArrayValue()); - ret->push_back(new Value("toplevel")); - ret->push_back(new ArrayValue()); + Ref ret(makeArray()); + ret->push_back(makeString("toplevel")); + ret->push_back(makeArray()); return ret; } Ref makePair(Ref x, Ref y) { - Ref ret = new ArrayValue(); + Ref ret = makeArray(); ret->push_back(x); ret->push_back(y); return ret; }; Ref makeNum(double x) { - Ref ret(new ArrayValue()); - ret->push_back(new Value("num")); - ret->push_back(new Value(x)); + Ref ret(makeArray()); + ret->push_back(makeString("num")); + ret->push_back(&arena.alloc()->setNumber(x)); return ret; } Ref makeName(const char *str) { - Ref ret(new ArrayValue()); - ret->push_back(new Value("name")); - ret->push_back(new Value(str)); + Ref ret(makeArray()); + ret->push_back(makeString("name")); + ret->push_back(makeString(str)); return ret; } @@ -421,47 +434,47 @@ Ref makeName(const std::string& str) { } Ref makeBlock() { - Ref ret(new ArrayValue()); - ret->push_back(new Value("block")); - ret->push_back(new ArrayValue()); + Ref ret(makeArray()); + ret->push_back(makeString("block")); + ret->push_back(makeArray()); return ret; } Ref make1(const char* type, Ref a) { - Ref ret(new ArrayValue()); - ret->push_back(new Value(type)); + Ref ret(makeArray()); + ret->push_back(makeString(type)); ret->push_back(a); return ret; } Ref make2(const char* type, const char *a, Ref b) { - Ref ret(new ArrayValue()); - ret->push_back(new Value(type)); - ret->push_back(new Value(a)); + Ref ret(makeArray()); + ret->push_back(makeString(type)); + ret->push_back(makeString(a)); ret->push_back(b); return ret; } Ref make2(const char* type, Ref a, Ref b) { - Ref ret(new ArrayValue()); - ret->push_back(new Value(type)); + Ref ret(makeArray()); + ret->push_back(makeString(type)); ret->push_back(a); ret->push_back(b); return ret; } Ref make3(const char *type, const char *a, Ref b, Ref c) { - Ref ret(new ArrayValue()); - ret->push_back(new Value(type)); - ret->push_back(new Value(a)); + Ref ret(makeArray()); + ret->push_back(makeString(type)); + ret->push_back(makeString(a)); ret->push_back(b); ret->push_back(c); return ret; } Ref make3(const char *type, Ref a, Ref b, Ref c) { - Ref ret(new ArrayValue()); - ret->push_back(new Value(type)); + Ref ret(makeArray()); + ret->push_back(makeString(type)); ret->push_back(a); ret->push_back(b); ret->push_back(c); @@ -469,7 +482,7 @@ Ref make3(const char *type, Ref a, Ref b, Ref c) { } Ref makeAsmVarDef(const std::string& v_, AsmType type) { - Ref v = new Value(v_); + Ref v = makeString(v_); Ref val; switch (type) { case ASM_INT: val = makeNum(0); break; @@ -478,16 +491,16 @@ Ref makeAsmVarDef(const std::string& v_, AsmType type) { if (ASM_FLOAT_ZERO.size() > 0) { val = makeName(ASM_FLOAT_ZERO.c_str()); } else { - val = make2("call", makeName("Math_fround"), &(new ArrayValue())->push_back(makeNum(0))); + val = make2("call", makeName("Math_fround"), &(makeArray())->push_back(makeNum(0))); } break; } case ASM_FLOAT32X4: { - val = make2("call", makeName("SIMD_float32x4"), &(new ArrayValue())->push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0))); + val = make2("call", makeName("SIMD_float32x4"), &(makeArray())->push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0))); break; } case ASM_INT32X4: { - val = make2("call", makeName("SIMD_int32x4"), &(new ArrayValue())->push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0))); + val = make2("call", makeName("SIMD_int32x4"), &(makeArray())->push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0))); break; } default: assert(0); @@ -499,9 +512,9 @@ Ref makeAsmCoercion(Ref node, AsmType type) { switch (type) { case ASM_INT: return make3("binary", "|", node, makeNum(0)); case ASM_DOUBLE: return make2("unary-prefix", "+", node); - case ASM_FLOAT: return make2("call", makeName("Math_fround"), &(new ArrayValue())->push_back(node)); - case ASM_FLOAT32X4: return make2("call", makeName("SIMD_float32x4"), &(new ArrayValue())->push_back(node)); - case ASM_INT32X4: return make2("call", makeName("SIMD_int32x4"), &(new ArrayValue())->push_back(node)); + case ASM_FLOAT: return make2("call", makeName("Math_fround"), &(makeArray())->push_back(node)); + case ASM_FLOAT32X4: return make2("call", makeName("SIMD_float32x4"), &(makeArray())->push_back(node)); + case ASM_INT32X4: return make2("call", makeName("SIMD_int32x4"), &(makeArray())->push_back(node)); case ASM_NONE: default: return node; // non-validating code, emit nothing XXX this is dangerous, we should only allow this when we know we are not validating } @@ -1178,7 +1191,7 @@ void simplifyIfs(Ref ast) { Ref curr = deStat(stats[i]); other[1] = make2("seq", curr, other[1]); } - Ref temp = new ArrayValue(); + Ref temp = makeArray(); temp->push_back(other); stats = body[1] = temp; } @@ -1323,7 +1336,7 @@ int main(int argc, char **argv) { if (comment) *comment = 0; // drop off the comments; TODO: parse extra info // Parse JSON source into the document - doc = new Value(); + doc = arena.alloc(); doc->parse(json); delete[] json; From 0163724affe8d37a9e21f0dc5509ce270da72399 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Nov 2014 18:28:58 -0800 Subject: [PATCH 068/161] fix comment --- tools/optimizer/minijson.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index f4a1ceb524cb4..bffe14c552a08 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -25,7 +25,7 @@ struct Value; void dump(const char *str, Ref node, bool pretty=false); -// Reference to a value. Simple shared_ptr, plus [] operator for convenience - we work on lots of arrays +// Reference to a value, plus some operators for convenience struct Ref { Value* inst; From 5eb2cedc7da3e9fa03e683242f2efaf222f404af Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sat, 8 Nov 2014 10:40:35 -0800 Subject: [PATCH 069/161] remove rapidjson --- tools/optimizer/rapidjson/allocators.h | 246 --- tools/optimizer/rapidjson/document.h | 1908 ----------------- tools/optimizer/rapidjson/encodedstream.h | 290 --- tools/optimizer/rapidjson/encodings.h | 630 ------ tools/optimizer/rapidjson/error/en.h | 71 - tools/optimizer/rapidjson/error/error.h | 150 -- tools/optimizer/rapidjson/filereadstream.h | 94 - tools/optimizer/rapidjson/filestream.h | 73 - tools/optimizer/rapidjson/filewritestream.h | 97 - tools/optimizer/rapidjson/internal/dtoa.h | 418 ---- tools/optimizer/rapidjson/internal/itoa.h | 306 --- tools/optimizer/rapidjson/internal/meta.h | 189 -- tools/optimizer/rapidjson/internal/pow10.h | 59 - tools/optimizer/rapidjson/internal/stack.h | 183 -- tools/optimizer/rapidjson/internal/strfunc.h | 43 - tools/optimizer/rapidjson/memorybuffer.h | 76 - tools/optimizer/rapidjson/memorystream.h | 67 - .../optimizer/rapidjson/msinttypes/inttypes.h | 306 --- tools/optimizer/rapidjson/msinttypes/stdint.h | 296 --- tools/optimizer/rapidjson/prettywriter.h | 205 -- tools/optimizer/rapidjson/rapidjson.h | 567 ----- tools/optimizer/rapidjson/reader.h | 1369 ------------ tools/optimizer/rapidjson/stringbuffer.h | 79 - tools/optimizer/rapidjson/writer.h | 389 ---- 24 files changed, 8111 deletions(-) delete mode 100644 tools/optimizer/rapidjson/allocators.h delete mode 100644 tools/optimizer/rapidjson/document.h delete mode 100644 tools/optimizer/rapidjson/encodedstream.h delete mode 100644 tools/optimizer/rapidjson/encodings.h delete mode 100644 tools/optimizer/rapidjson/error/en.h delete mode 100644 tools/optimizer/rapidjson/error/error.h delete mode 100644 tools/optimizer/rapidjson/filereadstream.h delete mode 100644 tools/optimizer/rapidjson/filestream.h delete mode 100644 tools/optimizer/rapidjson/filewritestream.h delete mode 100644 tools/optimizer/rapidjson/internal/dtoa.h delete mode 100644 tools/optimizer/rapidjson/internal/itoa.h delete mode 100644 tools/optimizer/rapidjson/internal/meta.h delete mode 100644 tools/optimizer/rapidjson/internal/pow10.h delete mode 100644 tools/optimizer/rapidjson/internal/stack.h delete mode 100644 tools/optimizer/rapidjson/internal/strfunc.h delete mode 100644 tools/optimizer/rapidjson/memorybuffer.h delete mode 100644 tools/optimizer/rapidjson/memorystream.h delete mode 100644 tools/optimizer/rapidjson/msinttypes/inttypes.h delete mode 100644 tools/optimizer/rapidjson/msinttypes/stdint.h delete mode 100644 tools/optimizer/rapidjson/prettywriter.h delete mode 100644 tools/optimizer/rapidjson/rapidjson.h delete mode 100644 tools/optimizer/rapidjson/reader.h delete mode 100644 tools/optimizer/rapidjson/stringbuffer.h delete mode 100644 tools/optimizer/rapidjson/writer.h diff --git a/tools/optimizer/rapidjson/allocators.h b/tools/optimizer/rapidjson/allocators.h deleted file mode 100644 index c99485e5080f4..0000000000000 --- a/tools/optimizer/rapidjson/allocators.h +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_ALLOCATORS_H_ -#define RAPIDJSON_ALLOCATORS_H_ - -#include "rapidjson.h" - -namespace rapidjson { - -/////////////////////////////////////////////////////////////////////////////// -// Allocator - -/*! \class rapidjson::Allocator - \brief Concept for allocating, resizing and freeing memory block. - - Note that Malloc() and Realloc() are non-static but Free() is static. - - So if an allocator need to support Free(), it needs to put its pointer in - the header of memory block. - -\code -concept Allocator { - static const bool kNeedFree; //!< Whether this allocator needs to call Free(). - - // Allocate a memory block. - // \param size of the memory block in bytes. - // \returns pointer to the memory block. - void* Malloc(size_t size); - - // Resize a memory block. - // \param originalPtr The pointer to current memory block. Null pointer is permitted. - // \param originalSize The current size in bytes. (Design issue: since some allocator may not book-keep this, explicitly pass to it can save memory.) - // \param newSize the new size in bytes. - void* Realloc(void* originalPtr, size_t originalSize, size_t newSize); - - // Free a memory block. - // \param pointer to the memory block. Null pointer is permitted. - static void Free(void *ptr); -}; -\endcode -*/ - -/////////////////////////////////////////////////////////////////////////////// -// CrtAllocator - -//! C-runtime library allocator. -/*! This class is just wrapper for standard C library memory routines. - \note implements Allocator concept -*/ -class CrtAllocator { -public: - static const bool kNeedFree = true; - void* Malloc(size_t size) { return std::malloc(size); } - void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { (void)originalSize; return std::realloc(originalPtr, newSize); } - static void Free(void *ptr) { std::free(ptr); } -}; - -/////////////////////////////////////////////////////////////////////////////// -// MemoryPoolAllocator - -//! Default memory allocator used by the parser and DOM. -/*! This allocator allocate memory blocks from pre-allocated memory chunks. - - It does not free memory blocks. And Realloc() only allocate new memory. - - The memory chunks are allocated by BaseAllocator, which is CrtAllocator by default. - - User may also supply a buffer as the first chunk. - - If the user-buffer is full then additional chunks are allocated by BaseAllocator. - - The user-buffer is not deallocated by this allocator. - - \tparam BaseAllocator the allocator type for allocating memory chunks. Default is CrtAllocator. - \note implements Allocator concept -*/ -template -class MemoryPoolAllocator { -public: - static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) - - //! Constructor with chunkSize. - /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. - \param baseAllocator The allocator for allocating memory chunks. - */ - MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : - chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0) - { - if (!baseAllocator_) - ownBaseAllocator_ = baseAllocator_ = new BaseAllocator(); - AddChunk(chunk_capacity_); - } - - //! Constructor with user-supplied buffer. - /*! The user buffer will be used firstly. When it is full, memory pool allocates new chunk with chunk size. - - The user buffer will not be deallocated when this allocator is destructed. - - \param buffer User supplied buffer. - \param size Size of the buffer in bytes. It must at least larger than sizeof(ChunkHeader). - \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. - \param baseAllocator The allocator for allocating memory chunks. - */ - MemoryPoolAllocator(void *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : - chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0) - { - RAPIDJSON_ASSERT(buffer != 0); - RAPIDJSON_ASSERT(size > sizeof(ChunkHeader)); - chunkHead_ = reinterpret_cast(buffer); - chunkHead_->capacity = size - sizeof(ChunkHeader); - chunkHead_->size = 0; - chunkHead_->next = 0; - } - - //! Destructor. - /*! This deallocates all memory chunks, excluding the user-supplied buffer. - */ - ~MemoryPoolAllocator() { - Clear(); - delete ownBaseAllocator_; - } - - //! Deallocates all memory chunks, excluding the user-supplied buffer. - void Clear() { - while(chunkHead_ != 0 && chunkHead_ != userBuffer_) { - ChunkHeader* next = chunkHead_->next; - baseAllocator_->Free(chunkHead_); - chunkHead_ = next; - } - } - - //! Computes the total capacity of allocated memory chunks. - /*! \return total capacity in bytes. - */ - size_t Capacity() const { - size_t capacity = 0; - for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) - capacity += c->capacity; - return capacity; - } - - //! Computes the memory blocks allocated. - /*! \return total used bytes. - */ - size_t Size() const { - size_t size = 0; - for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) - size += c->size; - return size; - } - - //! Allocates a memory block. (concept Allocator) - void* Malloc(size_t size) { - size = RAPIDJSON_ALIGN(size); - if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) - AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size); - - void *buffer = reinterpret_cast(chunkHead_ + 1) + chunkHead_->size; - chunkHead_->size += size; - return buffer; - } - - //! Resizes a memory block (concept Allocator) - void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { - if (originalPtr == 0) - return Malloc(newSize); - - // Do not shrink if new size is smaller than original - if (originalSize >= newSize) - return originalPtr; - - // Simply expand it if it is the last allocation and there is sufficient space - if (originalPtr == (char *)(chunkHead_ + 1) + chunkHead_->size - originalSize) { - size_t increment = static_cast(newSize - originalSize); - increment = RAPIDJSON_ALIGN(increment); - if (chunkHead_->size + increment <= chunkHead_->capacity) { - chunkHead_->size += increment; - return originalPtr; - } - } - - // Realloc process: allocate and copy memory, do not free original buffer. - void* newBuffer = Malloc(newSize); - RAPIDJSON_ASSERT(newBuffer != 0); // Do not handle out-of-memory explicitly. - return std::memcpy(newBuffer, originalPtr, originalSize); - } - - //! Frees a memory block (concept Allocator) - static void Free(void *ptr) { (void)ptr; } // Do nothing - -private: - //! Copy constructor is not permitted. - MemoryPoolAllocator(const MemoryPoolAllocator& rhs) /* = delete */; - //! Copy assignment operator is not permitted. - MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) /* = delete */; - - //! Creates a new chunk. - /*! \param capacity Capacity of the chunk in bytes. - */ - void AddChunk(size_t capacity) { - ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(sizeof(ChunkHeader) + capacity)); - chunk->capacity = capacity; - chunk->size = 0; - chunk->next = chunkHead_; - chunkHead_ = chunk; - } - - static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity. - - //! Chunk header for perpending to each chunk. - /*! Chunks are stored as a singly linked list. - */ - struct ChunkHeader { - size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). - size_t size; //!< Current size of allocated memory in bytes. - ChunkHeader *next; //!< Next chunk in the linked list. - }; - - ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation. - size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. - void *userBuffer_; //!< User supplied buffer. - BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. - BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object. -}; - -} // namespace rapidjson - -#endif // RAPIDJSON_ENCODINGS_H_ diff --git a/tools/optimizer/rapidjson/document.h b/tools/optimizer/rapidjson/document.h deleted file mode 100644 index 0f7974738f6b6..0000000000000 --- a/tools/optimizer/rapidjson/document.h +++ /dev/null @@ -1,1908 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_DOCUMENT_H_ -#define RAPIDJSON_DOCUMENT_H_ - -/*! \file document.h */ - -#include "reader.h" -#include "internal/meta.h" -#include "internal/strfunc.h" -#include // placement new - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant -#elif defined(__GNUC__) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_HAS_STDSTRING - -#ifndef RAPIDJSON_HAS_STDSTRING -#ifdef RAPIDJSON_DOXYGEN_RUNNING -#define RAPIDJSON_HAS_STDSTRING 1 // force generation of documentation -#else -#define RAPIDJSON_HAS_STDSTRING 0 // no std::string support by default -#endif -/*! \def RAPIDJSON_HAS_STDSTRING - \ingroup RAPIDJSON_CONFIG - \brief Enable RapidJSON support for \c std::string - - By defining this preprocessor symbol to \c 1, several convenience functions for using - \ref rapidjson::GenericValue with \c std::string are enabled, especially - for construction and comparison. - - \hideinitializer -*/ -#include -#endif // RAPIDJSON_HAS_STDSTRING - -#ifndef RAPIDJSON_NOMEMBERITERATORCLASS -#include // std::iterator, std::random_access_iterator_tag -#endif - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS -#include // std::move -#endif - -namespace rapidjson { - -// Forward declaration. -template -class GenericValue; - -//! Name-value pair in a JSON object value. -/*! - This class was internal to GenericValue. It used to be a inner struct. - But a compiler (IBM XL C/C++ for AIX) have reported to have problem with that so it moved as a namespace scope struct. - https://code.google.com/p/rapidjson/issues/detail?id=64 -*/ -template -struct GenericMember { - GenericValue name; //!< name of member (must be a string) - GenericValue value; //!< value of member. -}; - -/////////////////////////////////////////////////////////////////////////////// -// GenericMemberIterator - -#ifndef RAPIDJSON_NOMEMBERITERATORCLASS - -//! (Constant) member iterator for a JSON object value -/*! - \tparam Const Is this a constant iterator? - \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) - \tparam Allocator Allocator type for allocating memory of object, array and string. - - This class implements a Random Access Iterator for GenericMember elements - of a GenericValue, see ISO/IEC 14882:2003(E) C++ standard, 24.1 [lib.iterator.requirements]. - - \note This iterator implementation is mainly intended to avoid implicit - conversions from iterator values to \c NULL, - e.g. from GenericValue::FindMember. - - \note Define \c RAPIDJSON_NOMEMBERITERATORCLASS to fall back to a - pointer-based implementation, if your platform doesn't provide - the C++ header. - - \see GenericMember, GenericValue::MemberIterator, GenericValue::ConstMemberIterator - */ -template -class GenericMemberIterator - : public std::iterator >::Type> { - - friend class GenericValue; - template friend class GenericMemberIterator; - - typedef GenericMember PlainType; - typedef typename internal::MaybeAddConst::Type ValueType; - typedef std::iterator BaseType; - -public: - //! Iterator type itself - typedef GenericMemberIterator Iterator; - //! Constant iterator type - typedef GenericMemberIterator ConstIterator; - //! Non-constant iterator type - typedef GenericMemberIterator NonConstIterator; - - //! Pointer to (const) GenericMember - typedef typename BaseType::pointer Pointer; - //! Reference to (const) GenericMember - typedef typename BaseType::reference Reference; - //! Signed integer type (e.g. \c ptrdiff_t) - typedef typename BaseType::difference_type DifferenceType; - - //! Default constructor (singular value) - /*! Creates an iterator pointing to no element. - \note All operations, except for comparisons, are undefined on such values. - */ - GenericMemberIterator() : ptr_() {} - - //! Iterator conversions to more const - /*! - \param it (Non-const) iterator to copy from - - Allows the creation of an iterator from another GenericMemberIterator - that is "less const". Especially, creating a non-constant iterator - from a constant iterator are disabled: - \li const -> non-const (not ok) - \li const -> const (ok) - \li non-const -> const (ok) - \li non-const -> non-const (ok) - - \note If the \c Const template parameter is already \c false, this - constructor effectively defines a regular copy-constructor. - Otherwise, the copy constructor is implicitly defined. - */ - GenericMemberIterator(const NonConstIterator & it) : ptr_(it.ptr_) {} - - //! @name stepping - //@{ - Iterator& operator++(){ ++ptr_; return *this; } - Iterator& operator--(){ --ptr_; return *this; } - Iterator operator++(int){ Iterator old(*this); ++ptr_; return old; } - Iterator operator--(int){ Iterator old(*this); --ptr_; return old; } - //@} - - //! @name increment/decrement - //@{ - Iterator operator+(DifferenceType n) const { return Iterator(ptr_+n); } - Iterator operator-(DifferenceType n) const { return Iterator(ptr_-n); } - - Iterator& operator+=(DifferenceType n) { ptr_+=n; return *this; } - Iterator& operator-=(DifferenceType n) { ptr_-=n; return *this; } - //@} - - //! @name relations - //@{ - bool operator==(ConstIterator that) const { return ptr_ == that.ptr_; } - bool operator!=(ConstIterator that) const { return ptr_ != that.ptr_; } - bool operator<=(ConstIterator that) const { return ptr_ <= that.ptr_; } - bool operator>=(ConstIterator that) const { return ptr_ >= that.ptr_; } - bool operator< (ConstIterator that) const { return ptr_ < that.ptr_; } - bool operator> (ConstIterator that) const { return ptr_ > that.ptr_; } - //@} - - //! @name dereference - //@{ - Reference operator*() const { return *ptr_; } - Pointer operator->() const { return ptr_; } - Reference operator[](DifferenceType n) const { return ptr_[n]; } - //@} - - //! Distance - DifferenceType operator-(ConstIterator that) const { return ptr_-that.ptr_; } - -private: - //! Internal constructor from plain pointer - explicit GenericMemberIterator(Pointer p) : ptr_(p) {} - - Pointer ptr_; //!< raw pointer -}; - -#else // RAPIDJSON_NOMEMBERITERATORCLASS - -// class-based member iterator implementation disabled, use plain pointers - -template -struct GenericMemberIterator; - -//! non-const GenericMemberIterator -template -struct GenericMemberIterator { - //! use plain pointer as iterator type - typedef GenericMember* Iterator; -}; -//! const GenericMemberIterator -template -struct GenericMemberIterator { - //! use plain const pointer as iterator type - typedef const GenericMember* Iterator; -}; - -#endif // RAPIDJSON_NOMEMBERITERATORCLASS - -/////////////////////////////////////////////////////////////////////////////// -// GenericStringRef - -//! Reference to a constant string (not taking a copy) -/*! - \tparam CharType character type of the string - - This helper class is used to automatically infer constant string - references for string literals, especially from \c const \b (!) - character arrays. - - The main use is for creating JSON string values without copying the - source string via an \ref Allocator. This requires that the referenced - string pointers have a sufficient lifetime, which exceeds the lifetime - of the associated GenericValue. - - \b Example - \code - Value v("foo"); // ok, no need to copy & calculate length - const char foo[] = "foo"; - v.SetString(foo); // ok - - const char* bar = foo; - // Value x(bar); // not ok, can't rely on bar's lifetime - Value x(StringRef(bar)); // lifetime explicitly guaranteed by user - Value y(StringRef(bar, 3)); // ok, explicitly pass length - \endcode - - \see StringRef, GenericValue::SetString -*/ -template -struct GenericStringRef { - typedef CharType Ch; //!< character type of the string - - //! Create string reference from \c const character array - /*! - This constructor implicitly creates a constant string reference from - a \c const character array. It has better performance than - \ref StringRef(const CharType*) by inferring the string \ref length - from the array length, and also supports strings containing null - characters. - - \tparam N length of the string, automatically inferred - - \param str Constant character array, lifetime assumed to be longer - than the use of the string in e.g. a GenericValue - - \post \ref s == str - - \note Constant complexity. - \note There is a hidden, private overload to disallow references to - non-const character arrays to be created via this constructor. - By this, e.g. function-scope arrays used to be filled via - \c snprintf are excluded from consideration. - In such cases, the referenced string should be \b copied to the - GenericValue instead. - */ - template - GenericStringRef(const CharType (&str)[N]) RAPIDJSON_NOEXCEPT - : s(str), length(N-1) {} - - //! Explicitly create string reference from \c const character pointer - /*! - This constructor can be used to \b explicitly create a reference to - a constant string pointer. - - \see StringRef(const CharType*) - - \param str Constant character pointer, lifetime assumed to be longer - than the use of the string in e.g. a GenericValue - - \post \ref s == str - - \note There is a hidden, private overload to disallow references to - non-const character arrays to be created via this constructor. - By this, e.g. function-scope arrays used to be filled via - \c snprintf are excluded from consideration. - In such cases, the referenced string should be \b copied to the - GenericValue instead. - */ - explicit GenericStringRef(const CharType* str) - : s(str), length(internal::StrLen(str)){ RAPIDJSON_ASSERT(s != NULL); } - - //! Create constant string reference from pointer and length - /*! \param str constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue - \param len length of the string, excluding the trailing NULL terminator - - \post \ref s == str && \ref length == len - \note Constant complexity. - */ - GenericStringRef(const CharType* str, SizeType len) - : s(str), length(len) { RAPIDJSON_ASSERT(s != NULL); } - - //! implicit conversion to plain CharType pointer - operator const Ch *() const { return s; } - - const Ch* const s; //!< plain CharType pointer - const SizeType length; //!< length of the string (excluding the trailing NULL terminator) - -private: - //! Disallow copy-assignment - GenericStringRef operator=(const GenericStringRef&); - //! Disallow construction from non-const array - template - GenericStringRef(CharType (&str)[N]) /* = delete */; -}; - -//! Mark a character pointer as constant string -/*! Mark a plain character pointer as a "string literal". This function - can be used to avoid copying a character string to be referenced as a - value in a JSON GenericValue object, if the string's lifetime is known - to be valid long enough. - \tparam CharType Character type of the string - \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue - \return GenericStringRef string reference object - \relatesalso GenericStringRef - - \see GenericValue::GenericValue(StringRefType), GenericValue::operator=(StringRefType), GenericValue::SetString(StringRefType), GenericValue::PushBack(StringRefType, Allocator&), GenericValue::AddMember -*/ -template -inline GenericStringRef StringRef(const CharType* str) { - return GenericStringRef(str, internal::StrLen(str)); -} - -//! Mark a character pointer as constant string -/*! Mark a plain character pointer as a "string literal". This function - can be used to avoid copying a character string to be referenced as a - value in a JSON GenericValue object, if the string's lifetime is known - to be valid long enough. - - This version has better performance with supplied length, and also - supports string containing null characters. - - \tparam CharType character type of the string - \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue - \param length The length of source string. - \return GenericStringRef string reference object - \relatesalso GenericStringRef -*/ -template -inline GenericStringRef StringRef(const CharType* str, size_t length) { - return GenericStringRef(str, SizeType(length)); -} - -#if RAPIDJSON_HAS_STDSTRING -//! Mark a string object as constant string -/*! Mark a string object (e.g. \c std::string) as a "string literal". - This function can be used to avoid copying a string to be referenced as a - value in a JSON GenericValue object, if the string's lifetime is known - to be valid long enough. - - \tparam CharType character type of the string - \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue - \return GenericStringRef string reference object - \relatesalso GenericStringRef - \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. -*/ -template -inline GenericStringRef StringRef(const std::basic_string& str) { - return GenericStringRef(str.data(), SizeType(str.size())); -} -#endif - -/////////////////////////////////////////////////////////////////////////////// -// GenericValue type traits -namespace internal { - -template -struct IsGenericValueImpl : FalseType {}; - -// select candidates according to nested encoding and allocator types -template struct IsGenericValueImpl::Type, typename Void::Type> - : IsBaseOf, T>::Type {}; - -// helper to match arbitrary GenericValue instantiations, including derived classes -template struct IsGenericValue : IsGenericValueImpl::Type {}; - -} // namespace internal - -/////////////////////////////////////////////////////////////////////////////// -// GenericValue - -//! Represents a JSON value. Use Value for UTF8 encoding and default allocator. -/*! - A JSON value can be one of 7 types. This class is a variant type supporting - these types. - - Use the Value if UTF8 and default allocator - - \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) - \tparam Allocator Allocator type for allocating memory of object, array and string. -*/ -template > -class GenericValue { -public: - //! Name-value pair in an object. - typedef GenericMember Member; - typedef Encoding EncodingType; //!< Encoding type from template parameter. - typedef Allocator AllocatorType; //!< Allocator type from template parameter. - typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. - typedef GenericStringRef StringRefType; //!< Reference to a constant string - typedef typename GenericMemberIterator::Iterator MemberIterator; //!< Member iterator for iterating in object. - typedef typename GenericMemberIterator::Iterator ConstMemberIterator; //!< Constant member iterator for iterating in object. - typedef GenericValue* ValueIterator; //!< Value iterator for iterating in array. - typedef const GenericValue* ConstValueIterator; //!< Constant value iterator for iterating in array. - - //!@name Constructors and destructor. - //@{ - - //! Default constructor creates a null value. - GenericValue() RAPIDJSON_NOEXCEPT : data_(), flags_(kNullFlag) {} - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move constructor in C++11 - GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_), flags_(rhs.flags_) { - rhs.flags_ = kNullFlag; // give up contents - } -#endif - -private: - //! Copy constructor is not permitted. - GenericValue(const GenericValue& rhs); - -public: - - //! Constructor with JSON value type. - /*! This creates a Value of specified type with default content. - \param type Type of the value. - \note Default content for number is zero. - */ - explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_(), flags_() { - static const unsigned defaultFlags[7] = { - kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kConstStringFlag, - kNumberAnyFlag - }; - RAPIDJSON_ASSERT(type <= kNumberType); - flags_ = defaultFlags[type]; - } - - //! Explicit copy constructor (with allocator) - /*! Creates a copy of a Value by using the given Allocator - \tparam SourceAllocator allocator of \c rhs - \param rhs Value to copy from (read-only) - \param allocator Allocator for allocating copied elements and buffers. Commonly use GenericDocument::GetAllocator(). - \see CopyFrom() - */ - template< typename SourceAllocator > - GenericValue(const GenericValue& rhs, Allocator & allocator); - - //! Constructor for boolean value. - /*! \param b Boolean value - \note This constructor is limited to \em real boolean values and rejects - implicitly converted types like arbitrary pointers. Use an explicit cast - to \c bool, if you want to construct a boolean JSON value in such cases. - */ -#ifndef RAPIDJSON_DOXYGEN_RUNNING // hide SFINAE from Doxygen - template - explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame))) RAPIDJSON_NOEXCEPT -#else - explicit GenericValue(bool b) RAPIDJSON_NOEXCEPT -#endif - : data_(), flags_(b ? kTrueFlag : kFalseFlag) { - // safe-guard against failing SFINAE - RAPIDJSON_STATIC_ASSERT((internal::IsSame::Value)); - } - - //! Constructor for int value. - explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberIntFlag) { - data_.n.i64 = i; - if (i >= 0) - flags_ |= kUintFlag | kUint64Flag; - } - - //! Constructor for unsigned value. - explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberUintFlag) { - data_.n.u64 = u; - if (!(u & 0x80000000)) - flags_ |= kIntFlag | kInt64Flag; - } - - //! Constructor for int64_t value. - explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberInt64Flag) { - data_.n.i64 = i64; - if (i64 >= 0) { - flags_ |= kNumberUint64Flag; - if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) - flags_ |= kUintFlag; - if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - flags_ |= kIntFlag; - } - else if (i64 >= static_cast(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - flags_ |= kIntFlag; - } - - //! Constructor for uint64_t value. - explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberUint64Flag) { - data_.n.u64 = u64; - if (!(u64 & RAPIDJSON_UINT64_C2(0x80000000, 0x00000000))) - flags_ |= kInt64Flag; - if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) - flags_ |= kUintFlag; - if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - flags_ |= kIntFlag; - } - - //! Constructor for double value. - explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberDoubleFlag) { data_.n.d = d; } - - //! Constructor for constant string (i.e. do not make a copy of string) - GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_(), flags_() { SetStringRaw(StringRef(s, length)); } - - //! Constructor for constant string (i.e. do not make a copy of string) - explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_(), flags_() { SetStringRaw(s); } - - //! Constructor for copy-string (i.e. do make a copy of string) - GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s, length), allocator); } - - //! Constructor for copy-string (i.e. do make a copy of string) - GenericValue(const Ch*s, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s), allocator); } - -#if RAPIDJSON_HAS_STDSTRING - //! Constructor for copy-string from a string object (i.e. do make a copy of string) - /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. - */ - GenericValue(const std::basic_string& s, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s), allocator); } -#endif - - //! Destructor. - /*! Need to destruct elements of array, members of object, or copy-string. - */ - ~GenericValue() { - if (Allocator::kNeedFree) { // Shortcut by Allocator's trait - switch(flags_) { - case kArrayFlag: - for (GenericValue* v = data_.a.elements; v != data_.a.elements + data_.a.size; ++v) - v->~GenericValue(); - Allocator::Free(data_.a.elements); - break; - - case kObjectFlag: - for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) - m->~Member(); - Allocator::Free(data_.o.members); - break; - - case kCopyStringFlag: - Allocator::Free(const_cast(data_.s.str)); - break; - - default: - break; // Do nothing for other types. - } - } - } - - //@} - - //!@name Assignment operators - //@{ - - //! Assignment with move semantics. - /*! \param rhs Source of the assignment. It will become a null value after assignment. - */ - GenericValue& operator=(GenericValue& rhs) RAPIDJSON_NOEXCEPT { - RAPIDJSON_ASSERT(this != &rhs); - this->~GenericValue(); - RawAssign(rhs); - return *this; - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move assignment in C++11 - GenericValue& operator=(GenericValue&& rhs) RAPIDJSON_NOEXCEPT { - return *this = rhs.Move(); - } -#endif - - //! Assignment of constant string reference (no copy) - /*! \param str Constant string reference to be assigned - \note This overload is needed to avoid clashes with the generic primitive type assignment overload below. - \see GenericStringRef, operator=(T) - */ - GenericValue& operator=(StringRefType str) RAPIDJSON_NOEXCEPT { - GenericValue s(str); - return *this = s; - } - - //! Assignment with primitive types. - /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t - \param value The value to be assigned. - - \note The source type \c T explicitly disallows all pointer types, - especially (\c const) \ref Ch*. This helps avoiding implicitly - referencing character strings with insufficient lifetime, use - \ref SetString(const Ch*, Allocator&) (for copying) or - \ref StringRef() (to explicitly mark the pointer as constant) instead. - All other pointer types would implicitly convert to \c bool, - use \ref SetBool() instead. - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::IsPointer), (GenericValue&)) - operator=(T value) { - GenericValue v(value); - return *this = v; - } - - //! Deep-copy assignment from Value - /*! Assigns a \b copy of the Value to the current Value object - \tparam SourceAllocator Allocator type of \c rhs - \param rhs Value to copy from (read-only) - \param allocator Allocator to use for copying - */ - template - GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator) { - RAPIDJSON_ASSERT((void*)this != (void const*)&rhs); - this->~GenericValue(); - new (this) GenericValue(rhs, allocator); - return *this; - } - - //! Exchange the contents of this value with those of other. - /*! - \param other Another value. - \note Constant complexity. - */ - GenericValue& Swap(GenericValue& other) RAPIDJSON_NOEXCEPT { - GenericValue temp; - temp.RawAssign(*this); - RawAssign(other); - other.RawAssign(temp); - return *this; - } - - //! Prepare Value for move semantics - /*! \return *this */ - GenericValue& Move() RAPIDJSON_NOEXCEPT { return *this; } - //@} - - //!@name Equal-to and not-equal-to operators - //@{ - //! Equal-to operator - /*! - \note If an object contains duplicated named member, comparing equality with any object is always \c false. - \note Linear time complexity (number of all values in the subtree and total lengths of all strings). - */ - template - bool operator==(const GenericValue& rhs) const { - typedef GenericValue RhsType; - if (GetType() != rhs.GetType()) - return false; - - switch (GetType()) { - case kObjectType: // Warning: O(n^2) inner-loop - if (data_.o.size != rhs.data_.o.size) - return false; - for (ConstMemberIterator lhsMemberItr = MemberBegin(); lhsMemberItr != MemberEnd(); ++lhsMemberItr) { - typename RhsType::ConstMemberIterator rhsMemberItr = rhs.FindMember(lhsMemberItr->name); - if (rhsMemberItr == rhs.MemberEnd() || lhsMemberItr->value != rhsMemberItr->value) - return false; - } - return true; - - case kArrayType: - if (data_.a.size != rhs.data_.a.size) - return false; - for (SizeType i = 0; i < data_.a.size; i++) - if ((*this)[i] != rhs[i]) - return false; - return true; - - case kStringType: - return StringEqual(rhs); - - case kNumberType: - if (IsDouble() || rhs.IsDouble()) - return GetDouble() == rhs.GetDouble(); // May convert one operand from integer to double. - else - return data_.n.u64 == rhs.data_.n.u64; - - default: // kTrueType, kFalseType, kNullType - return true; - } - } - - //! Equal-to operator with const C-string pointer - bool operator==(const Ch* rhs) const { return *this == GenericValue(StringRef(rhs)); } - -#if RAPIDJSON_HAS_STDSTRING - //! Equal-to operator with string object - /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. - */ - bool operator==(const std::basic_string& rhs) const { return *this == GenericValue(StringRef(rhs)); } -#endif - - //! Equal-to operator with primitive types - /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c true, \c false - */ - template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr,internal::IsGenericValue >), (bool)) operator==(const T& rhs) const { return *this == GenericValue(rhs); } - - //! Not-equal-to operator - /*! \return !(*this == rhs) - */ - template - bool operator!=(const GenericValue& rhs) const { return !(*this == rhs); } - - //! Not-equal-to operator with const C-string pointer - bool operator!=(const Ch* rhs) const { return !(*this == rhs); } - - //! Not-equal-to operator with arbitrary types - /*! \return !(*this == rhs) - */ - template RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& rhs) const { return !(*this == rhs); } - - //! Equal-to operator with arbitrary types (symmetric version) - /*! \return (rhs == lhs) - */ - template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator==(const T& lhs, const GenericValue& rhs) { return rhs == lhs; } - - //! Not-Equal-to operator with arbitrary types (symmetric version) - /*! \return !(rhs == lhs) - */ - template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& lhs, const GenericValue& rhs) { return !(rhs == lhs); } - //@} - - //!@name Type - //@{ - - Type GetType() const { return static_cast(flags_ & kTypeMask); } - bool IsNull() const { return flags_ == kNullFlag; } - bool IsFalse() const { return flags_ == kFalseFlag; } - bool IsTrue() const { return flags_ == kTrueFlag; } - bool IsBool() const { return (flags_ & kBoolFlag) != 0; } - bool IsObject() const { return flags_ == kObjectFlag; } - bool IsArray() const { return flags_ == kArrayFlag; } - bool IsNumber() const { return (flags_ & kNumberFlag) != 0; } - bool IsInt() const { return (flags_ & kIntFlag) != 0; } - bool IsUint() const { return (flags_ & kUintFlag) != 0; } - bool IsInt64() const { return (flags_ & kInt64Flag) != 0; } - bool IsUint64() const { return (flags_ & kUint64Flag) != 0; } - bool IsDouble() const { return (flags_ & kDoubleFlag) != 0; } - bool IsString() const { return (flags_ & kStringFlag) != 0; } - - //@} - - //!@name Null - //@{ - - GenericValue& SetNull() { this->~GenericValue(); new (this) GenericValue(); return *this; } - - //@} - - //!@name Bool - //@{ - - bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return flags_ == kTrueFlag; } - //!< Set boolean value - /*! \post IsBool() == true */ - GenericValue& SetBool(bool b) { this->~GenericValue(); new (this) GenericValue(b); return *this; } - - //@} - - //!@name Object - //@{ - - //! Set this value as an empty object. - /*! \post IsObject() == true */ - GenericValue& SetObject() { this->~GenericValue(); new (this) GenericValue(kObjectType); return *this; } - - //! Get the number of members in the object. - SizeType MemberCount() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size; } - - //! Check whether the object is empty. - bool ObjectEmpty() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size == 0; } - - //! Get a value from an object associated with the name. - /*! \pre IsObject() == true - \tparam T Either \c Ch or \c const \c Ch (template used for disambiguation with \ref operator[](SizeType)) - \note In version 0.1x, if the member is not found, this function returns a null value. This makes issue 7. - Since 0.2, if the name is not correct, it will assert. - If user is unsure whether a member exists, user should use HasMember() first. - A better approach is to use FindMember(). - \note Linear time complexity. - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(GenericValue&)) operator[](T* name) { - GenericValue n(StringRef(name)); - return (*this)[n]; - } - template - RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(const GenericValue&)) operator[](T* name) const { return const_cast(*this)[name]; } - - //! Get a value from an object associated with the name. - /*! \pre IsObject() == true - \tparam SourceAllocator Allocator of the \c name value - - \note Compared to \ref operator[](T*), this version is faster because it does not need a StrLen(). - And it can also handle strings with embedded null characters. - - \note Linear time complexity. - */ - template - GenericValue& operator[](const GenericValue& name) { - MemberIterator member = FindMember(name); - if (member != MemberEnd()) - return member->value; - else { - RAPIDJSON_ASSERT(false); // see above note - static GenericValue NullValue; - return NullValue; - } - } - template - const GenericValue& operator[](const GenericValue& name) const { return const_cast(*this)[name]; } - - //! Const member iterator - /*! \pre IsObject() == true */ - ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(data_.o.members); } - //! Const \em past-the-end member iterator - /*! \pre IsObject() == true */ - ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(data_.o.members + data_.o.size); } - //! Member iterator - /*! \pre IsObject() == true */ - MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(data_.o.members); } - //! \em Past-the-end member iterator - /*! \pre IsObject() == true */ - MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(data_.o.members + data_.o.size); } - - //! Check whether a member exists in the object. - /*! - \param name Member name to be searched. - \pre IsObject() == true - \return Whether a member with that name exists. - \note It is better to use FindMember() directly if you need the obtain the value as well. - \note Linear time complexity. - */ - bool HasMember(const Ch* name) const { return FindMember(name) != MemberEnd(); } - - //! Check whether a member exists in the object with GenericValue name. - /*! - This version is faster because it does not need a StrLen(). It can also handle string with null character. - \param name Member name to be searched. - \pre IsObject() == true - \return Whether a member with that name exists. - \note It is better to use FindMember() directly if you need the obtain the value as well. - \note Linear time complexity. - */ - template - bool HasMember(const GenericValue& name) const { return FindMember(name) != MemberEnd(); } - - //! Find member by name. - /*! - \param name Member name to be searched. - \pre IsObject() == true - \return Iterator to member, if it exists. - Otherwise returns \ref MemberEnd(). - - \note Earlier versions of Rapidjson returned a \c NULL pointer, in case - the requested member doesn't exist. For consistency with e.g. - \c std::map, this has been changed to MemberEnd() now. - \note Linear time complexity. - */ - MemberIterator FindMember(const Ch* name) { - GenericValue n(StringRef(name)); - return FindMember(n); - } - - ConstMemberIterator FindMember(const Ch* name) const { return const_cast(*this).FindMember(name); } - - //! Find member by name. - /*! - This version is faster because it does not need a StrLen(). It can also handle string with null character. - \param name Member name to be searched. - \pre IsObject() == true - \return Iterator to member, if it exists. - Otherwise returns \ref MemberEnd(). - - \note Earlier versions of Rapidjson returned a \c NULL pointer, in case - the requested member doesn't exist. For consistency with e.g. - \c std::map, this has been changed to MemberEnd() now. - \note Linear time complexity. - */ - template - MemberIterator FindMember(const GenericValue& name) { - RAPIDJSON_ASSERT(IsObject()); - RAPIDJSON_ASSERT(name.IsString()); - MemberIterator member = MemberBegin(); - for ( ; member != MemberEnd(); ++member) - if (name.StringEqual(member->name)) - break; - return member; - } - template ConstMemberIterator FindMember(const GenericValue& name) const { return const_cast(*this).FindMember(name); } - - //! Add a member (name-value pair) to the object. - /*! \param name A string value as name of member. - \param value Value of any type. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \note The ownership of \c name and \c value will be transferred to this object on success. - \pre IsObject() && name.IsString() - \post name.IsNull() && value.IsNull() - \note Amortized Constant time complexity. - */ - GenericValue& AddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { - RAPIDJSON_ASSERT(IsObject()); - RAPIDJSON_ASSERT(name.IsString()); - - Object& o = data_.o; - if (o.size >= o.capacity) { - if (o.capacity == 0) { - o.capacity = kDefaultObjectCapacity; - o.members = reinterpret_cast(allocator.Malloc(o.capacity * sizeof(Member))); - } - else { - SizeType oldCapacity = o.capacity; - o.capacity += (oldCapacity + 1) / 2; // grow by factor 1.5 - o.members = reinterpret_cast(allocator.Realloc(o.members, oldCapacity * sizeof(Member), o.capacity * sizeof(Member))); - } - } - o.members[o.size].name.RawAssign(name); - o.members[o.size].value.RawAssign(value); - o.size++; - return *this; - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericValue& AddMember(GenericValue&& name, GenericValue&& value, Allocator& allocator) { - return AddMember(name, value, allocator); - } - GenericValue& AddMember(GenericValue&& name, GenericValue& value, Allocator& allocator) { - return AddMember(name, value, allocator); - } - GenericValue& AddMember(GenericValue& name, GenericValue&& value, Allocator& allocator) { - return AddMember(name, value, allocator); - } - GenericValue& AddMember(StringRefType name, GenericValue&& value, Allocator& allocator) { - GenericValue n(name); - return AddMember(n, value, allocator); - } -#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS - - - //! Add a member (name-value pair) to the object. - /*! \param name A constant string reference as name of member. - \param value Value of any type. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \note The ownership of \c value will be transferred to this object on success. - \pre IsObject() - \post value.IsNull() - \note Amortized Constant time complexity. - */ - GenericValue& AddMember(StringRefType name, GenericValue& value, Allocator& allocator) { - GenericValue n(name); - return AddMember(n, value, allocator); - } - - //! Add a constant string value as member (name-value pair) to the object. - /*! \param name A constant string reference as name of member. - \param value constant string reference as value of member. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \pre IsObject() - \note This overload is needed to avoid clashes with the generic primitive type AddMember(StringRefType,T,Allocator&) overload below. - \note Amortized Constant time complexity. - */ - GenericValue& AddMember(StringRefType name, StringRefType value, Allocator& allocator) { - GenericValue v(value); - return AddMember(name, v, allocator); - } - - //! Add any primitive value as member (name-value pair) to the object. - /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t - \param name A constant string reference as name of member. - \param value Value of primitive type \c T as value of member - \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \pre IsObject() - - \note The source type \c T explicitly disallows all pointer types, - especially (\c const) \ref Ch*. This helps avoiding implicitly - referencing character strings with insufficient lifetime, use - \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref - AddMember(StringRefType, StringRefType, Allocator&). - All other pointer types would implicitly convert to \c bool, - use an explicit cast instead, if needed. - \note Amortized Constant time complexity. - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) - AddMember(StringRefType name, T value, Allocator& allocator) { - GenericValue n(name); - GenericValue v(value); - return AddMember(n, v, allocator); - } - - //! Remove all members in the object. - /*! This function do not deallocate memory in the object, i.e. the capacity is unchanged. - \note Linear time complexity. - */ - void RemoveAllMembers() { - RAPIDJSON_ASSERT(IsObject()); - for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) - m->~Member(); - data_.o.size = 0; - } - - //! Remove a member in object by its name. - /*! \param name Name of member to be removed. - \return Whether the member existed. - \note This function may reorder the object members. Use \ref - EraseMember(ConstMemberIterator) if you need to preserve the - relative order of the remaining members. - \note Linear time complexity. - */ - bool RemoveMember(const Ch* name) { - GenericValue n(StringRef(name)); - return RemoveMember(n); - } - - template - bool RemoveMember(const GenericValue& name) { - MemberIterator m = FindMember(name); - if (m != MemberEnd()) { - RemoveMember(m); - return true; - } - else - return false; - } - - //! Remove a member in object by iterator. - /*! \param m member iterator (obtained by FindMember() or MemberBegin()). - \return the new iterator after removal. - \note This function may reorder the object members. Use \ref - EraseMember(ConstMemberIterator) if you need to preserve the - relative order of the remaining members. - \note Constant time complexity. - */ - MemberIterator RemoveMember(MemberIterator m) { - RAPIDJSON_ASSERT(IsObject()); - RAPIDJSON_ASSERT(data_.o.size > 0); - RAPIDJSON_ASSERT(data_.o.members != 0); - RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); - - MemberIterator last(data_.o.members + (data_.o.size - 1)); - if (data_.o.size > 1 && m != last) { - // Move the last one to this place - *m = *last; - } - else { - // Only one left, just destroy - m->~Member(); - } - --data_.o.size; - return m; - } - - //! Remove a member from an object by iterator. - /*! \param pos iterator to the member to remove - \pre IsObject() == true && \ref MemberBegin() <= \c pos < \ref MemberEnd() - \return Iterator following the removed element. - If the iterator \c pos refers to the last element, the \ref MemberEnd() iterator is returned. - \note This function preserves the relative order of the remaining object - members. If you do not need this, use the more efficient \ref RemoveMember(MemberIterator). - \note Linear time complexity. - */ - MemberIterator EraseMember(ConstMemberIterator pos) { - return EraseMember(pos, pos +1); - } - - //! Remove members in the range [first, last) from an object. - /*! \param first iterator to the first member to remove - \param last iterator following the last member to remove - \pre IsObject() == true && \ref MemberBegin() <= \c first <= \c last <= \ref MemberEnd() - \return Iterator following the last removed element. - \note This function preserves the relative order of the remaining object - members. - \note Linear time complexity. - */ - MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) { - RAPIDJSON_ASSERT(IsObject()); - RAPIDJSON_ASSERT(data_.o.size > 0); - RAPIDJSON_ASSERT(data_.o.members != 0); - RAPIDJSON_ASSERT(first >= MemberBegin()); - RAPIDJSON_ASSERT(first <= last); - RAPIDJSON_ASSERT(last <= MemberEnd()); - - MemberIterator pos = MemberBegin() + (first - MemberBegin()); - for (MemberIterator itr = pos; itr != last; ++itr) - itr->~Member(); - std::memmove(&*pos, &*last, (MemberEnd() - last) * sizeof(Member)); - data_.o.size -= (last - first); - return pos; - } - - //@} - - //!@name Array - //@{ - - //! Set this value as an empty array. - /*! \post IsArray == true */ - GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } - - //! Get the number of elements in array. - SizeType Size() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size; } - - //! Get the capacity of array. - SizeType Capacity() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.capacity; } - - //! Check whether the array is empty. - bool Empty() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size == 0; } - - //! Remove all elements in the array. - /*! This function do not deallocate memory in the array, i.e. the capacity is unchanged. - \note Linear time complexity. - */ - void Clear() { - RAPIDJSON_ASSERT(IsArray()); - for (SizeType i = 0; i < data_.a.size; ++i) - data_.a.elements[i].~GenericValue(); - data_.a.size = 0; - } - - //! Get an element from array by index. - /*! \pre IsArray() == true - \param index Zero-based index of element. - \see operator[](T*) - */ - GenericValue& operator[](SizeType index) { - RAPIDJSON_ASSERT(IsArray()); - RAPIDJSON_ASSERT(index < data_.a.size); - return data_.a.elements[index]; - } - const GenericValue& operator[](SizeType index) const { return const_cast(*this)[index]; } - - //! Element iterator - /*! \pre IsArray() == true */ - ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return data_.a.elements; } - //! \em Past-the-end element iterator - /*! \pre IsArray() == true */ - ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return data_.a.elements + data_.a.size; } - //! Constant element iterator - /*! \pre IsArray() == true */ - ConstValueIterator Begin() const { return const_cast(*this).Begin(); } - //! Constant \em past-the-end element iterator - /*! \pre IsArray() == true */ - ConstValueIterator End() const { return const_cast(*this).End(); } - - //! Request the array to have enough capacity to store elements. - /*! \param newCapacity The capacity that the array at least need to have. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \note Linear time complexity. - */ - GenericValue& Reserve(SizeType newCapacity, Allocator &allocator) { - RAPIDJSON_ASSERT(IsArray()); - if (newCapacity > data_.a.capacity) { - data_.a.elements = (GenericValue*)allocator.Realloc(data_.a.elements, data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)); - data_.a.capacity = newCapacity; - } - return *this; - } - - //! Append a GenericValue at the end of the array. - /*! \param value Value to be appended. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \pre IsArray() == true - \post value.IsNull() == true - \return The value itself for fluent API. - \note The ownership of \c value will be transferred to this array on success. - \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. - \note Amortized constant time complexity. - */ - GenericValue& PushBack(GenericValue& value, Allocator& allocator) { - RAPIDJSON_ASSERT(IsArray()); - if (data_.a.size >= data_.a.capacity) - Reserve(data_.a.capacity == 0 ? kDefaultArrayCapacity : (data_.a.capacity + (data_.a.capacity + 1) / 2), allocator); - data_.a.elements[data_.a.size++].RawAssign(value); - return *this; - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericValue& PushBack(GenericValue&& value, Allocator& allocator) { - return PushBack(value, allocator); - } -#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS - - //! Append a constant string reference at the end of the array. - /*! \param value Constant string reference to be appended. - \param allocator Allocator for reallocating memory. It must be the same one used previously. Commonly use GenericDocument::GetAllocator(). - \pre IsArray() == true - \return The value itself for fluent API. - \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. - \note Amortized constant time complexity. - \see GenericStringRef - */ - GenericValue& PushBack(StringRefType value, Allocator& allocator) { - return (*this).template PushBack(value, allocator); - } - - //! Append a primitive value at the end of the array. - /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t - \param value Value of primitive type T to be appended. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \pre IsArray() == true - \return The value itself for fluent API. - \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. - - \note The source type \c T explicitly disallows all pointer types, - especially (\c const) \ref Ch*. This helps avoiding implicitly - referencing character strings with insufficient lifetime, use - \ref PushBack(GenericValue&, Allocator&) or \ref - PushBack(StringRefType, Allocator&). - All other pointer types would implicitly convert to \c bool, - use an explicit cast instead, if needed. - \note Amortized constant time complexity. - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) - PushBack(T value, Allocator& allocator) { - GenericValue v(value); - return PushBack(v, allocator); - } - - //! Remove the last element in the array. - /*! - \note Constant time complexity. - */ - GenericValue& PopBack() { - RAPIDJSON_ASSERT(IsArray()); - RAPIDJSON_ASSERT(!Empty()); - data_.a.elements[--data_.a.size].~GenericValue(); - return *this; - } - - //! Remove an element of array by iterator. - /*! - \param pos iterator to the element to remove - \pre IsArray() == true && \ref Begin() <= \c pos < \ref End() - \return Iterator following the removed element. If the iterator pos refers to the last element, the End() iterator is returned. - \note Linear time complexity. - */ - ValueIterator Erase(ConstValueIterator pos) { - return Erase(pos, pos + 1); - } - - //! Remove elements in the range [first, last) of the array. - /*! - \param first iterator to the first element to remove - \param last iterator following the last element to remove - \pre IsArray() == true && \ref Begin() <= \c first <= \c last <= \ref End() - \return Iterator following the last removed element. - \note Linear time complexity. - */ - ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) { - RAPIDJSON_ASSERT(IsArray()); - RAPIDJSON_ASSERT(data_.a.size > 0); - RAPIDJSON_ASSERT(data_.a.elements != 0); - RAPIDJSON_ASSERT(first >= Begin()); - RAPIDJSON_ASSERT(first <= last); - RAPIDJSON_ASSERT(last <= End()); - ValueIterator pos = Begin() + (first - Begin()); - for (ValueIterator itr = pos; itr != last; ++itr) - itr->~GenericValue(); - std::memmove(pos, last, (End() - last) * sizeof(GenericValue)); - data_.a.size -= (last - first); - return pos; - } - - //@} - - //!@name Number - //@{ - - int GetInt() const { RAPIDJSON_ASSERT(flags_ & kIntFlag); return data_.n.i.i; } - unsigned GetUint() const { RAPIDJSON_ASSERT(flags_ & kUintFlag); return data_.n.u.u; } - int64_t GetInt64() const { RAPIDJSON_ASSERT(flags_ & kInt64Flag); return data_.n.i64; } - uint64_t GetUint64() const { RAPIDJSON_ASSERT(flags_ & kUint64Flag); return data_.n.u64; } - - double GetDouble() const { - RAPIDJSON_ASSERT(IsNumber()); - if ((flags_ & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. - if ((flags_ & kIntFlag) != 0) return data_.n.i.i; // int -> double - if ((flags_ & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double - if ((flags_ & kInt64Flag) != 0) return (double)data_.n.i64; // int64_t -> double (may lose precision) - RAPIDJSON_ASSERT((flags_ & kUint64Flag) != 0); return (double)data_.n.u64; // uint64_t -> double (may lose precision) - } - - GenericValue& SetInt(int i) { this->~GenericValue(); new (this) GenericValue(i); return *this; } - GenericValue& SetUint(unsigned u) { this->~GenericValue(); new (this) GenericValue(u); return *this; } - GenericValue& SetInt64(int64_t i64) { this->~GenericValue(); new (this) GenericValue(i64); return *this; } - GenericValue& SetUint64(uint64_t u64) { this->~GenericValue(); new (this) GenericValue(u64); return *this; } - GenericValue& SetDouble(double d) { this->~GenericValue(); new (this) GenericValue(d); return *this; } - - //@} - - //!@name String - //@{ - - const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return ((flags_ & kInlineStrFlag) ? data_.ss.str : data_.s.str); } - - //! Get the length of string. - /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). - */ - SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((flags_ & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } - - //! Set this value as a string without copying source string. - /*! This version has better performance with supplied length, and also support string containing null character. - \param s source string pointer. - \param length The length of source string, excluding the trailing null terminator. - \return The value itself for fluent API. - \post IsString() == true && GetString() == s && GetStringLength() == length - \see SetString(StringRefType) - */ - GenericValue& SetString(const Ch* s, SizeType length) { return SetString(StringRef(s, length)); } - - //! Set this value as a string without copying source string. - /*! \param s source string reference - \return The value itself for fluent API. - \post IsString() == true && GetString() == s && GetStringLength() == s.length - */ - GenericValue& SetString(StringRefType s) { this->~GenericValue(); SetStringRaw(s); return *this; } - - //! Set this value as a string by copying from source string. - /*! This version has better performance with supplied length, and also support string containing null character. - \param s source string. - \param length The length of source string, excluding the trailing null terminator. - \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length - */ - GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { this->~GenericValue(); SetStringRaw(StringRef(s, length), allocator); return *this; } - - //! Set this value as a string by copying from source string. - /*! \param s source string. - \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length - */ - GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(s, internal::StrLen(s), allocator); } - -#if RAPIDJSON_HAS_STDSTRING - //! Set this value as a string by copying from source string. - /*! \param s source string. - \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \post IsString() == true && GetString() != s.data() && strcmp(GetString(),s.data() == 0 && GetStringLength() == s.size() - \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. - */ - GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(s.data(), s.size(), allocator); } -#endif - - //@} - - //! Generate events of this value to a Handler. - /*! This function adopts the GoF visitor pattern. - Typical usage is to output this JSON value as JSON text via Writer, which is a Handler. - It can also be used to deep clone this value via GenericDocument, which is also a Handler. - \tparam Handler type of handler. - \param handler An object implementing concept Handler. - */ - template - bool Accept(Handler& handler) const { - switch(GetType()) { - case kNullType: return handler.Null(); - case kFalseType: return handler.Bool(false); - case kTrueType: return handler.Bool(true); - - case kObjectType: - if (!handler.StartObject()) - return false; - for (ConstMemberIterator m = MemberBegin(); m != MemberEnd(); ++m) { - if (!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.flags_ & kCopyFlag) != 0)) - return false; - if (!m->value.Accept(handler)) - return false; - } - return handler.EndObject(data_.o.size); - - case kArrayType: - if (!handler.StartArray()) - return false; - for (GenericValue* v = data_.a.elements; v != data_.a.elements + data_.a.size; ++v) - if (!v->Accept(handler)) - return false; - return handler.EndArray(data_.a.size); - - case kStringType: - return handler.String(GetString(), GetStringLength(), (flags_ & kCopyFlag) != 0); - - case kNumberType: - if (IsInt()) return handler.Int(data_.n.i.i); - else if (IsUint()) return handler.Uint(data_.n.u.u); - else if (IsInt64()) return handler.Int64(data_.n.i64); - else if (IsUint64()) return handler.Uint64(data_.n.u64); - else return handler.Double(data_.n.d); - - default: - RAPIDJSON_ASSERT(false); - } - return false; - } - -private: - template friend class GenericValue; - template friend class GenericDocument; - - enum { - kBoolFlag = 0x100, - kNumberFlag = 0x200, - kIntFlag = 0x400, - kUintFlag = 0x800, - kInt64Flag = 0x1000, - kUint64Flag = 0x2000, - kDoubleFlag = 0x4000, - kStringFlag = 0x100000, - kCopyFlag = 0x200000, - kInlineStrFlag = 0x400000, - - // Initial flags of different types. - kNullFlag = kNullType, - kTrueFlag = kTrueType | kBoolFlag, - kFalseFlag = kFalseType | kBoolFlag, - kNumberIntFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag, - kNumberUintFlag = kNumberType | kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag, - kNumberInt64Flag = kNumberType | kNumberFlag | kInt64Flag, - kNumberUint64Flag = kNumberType | kNumberFlag | kUint64Flag, - kNumberDoubleFlag = kNumberType | kNumberFlag | kDoubleFlag, - kNumberAnyFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag, - kConstStringFlag = kStringType | kStringFlag, - kCopyStringFlag = kStringType | kStringFlag | kCopyFlag, - kShortStringFlag = kStringType | kStringFlag | kCopyFlag | kInlineStrFlag, - kObjectFlag = kObjectType, - kArrayFlag = kArrayType, - - kTypeMask = 0xFF // bitwise-and with mask of 0xFF can be optimized by compiler - }; - - static const SizeType kDefaultArrayCapacity = 16; - static const SizeType kDefaultObjectCapacity = 16; - - struct String { - const Ch* str; - SizeType length; - unsigned hashcode; //!< reserved - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - // implementation detail: ShortString can represent zero-terminated strings up to MaxSize chars - // (excluding the terminating zero) and store a value to determine the length of the contained - // string in the last character str[LenPos] by storing "MaxSize - length" there. If the string - // to store has the maximal length of MaxSize then str[LenPos] will be 0 and therefore act as - // the string terminator as well. For getting the string length back from that value just use - // "MaxSize - str[LenPos]". - // This allows to store 11-chars strings in 32-bit mode and 15-chars strings in 64-bit mode - // inline (for `UTF8`-encoded strings). - struct ShortString { - enum { MaxChars = sizeof(String) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; - Ch str[MaxChars]; - - inline static bool Usable(SizeType len) { return (MaxSize >= len); } - inline void SetLength(SizeType len) { str[LenPos] = (Ch)(MaxSize - len); } - inline SizeType GetLength() const { return (SizeType)(MaxSize - str[LenPos]); } - }; // at most as many bytes as "String" above => 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - // By using proper binary layout, retrieval of different integer types do not need conversions. - union Number { -#if RAPIDJSON_ENDIAN == RAPIDJSON_LITTLEENDIAN - struct I { - int i; - char padding[4]; - }i; - struct U { - unsigned u; - char padding2[4]; - }u; -#else - struct I { - char padding[4]; - int i; - }i; - struct U { - char padding2[4]; - unsigned u; - }u; -#endif - int64_t i64; - uint64_t u64; - double d; - }; // 8 bytes - - struct Object { - Member* members; - SizeType size; - SizeType capacity; - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - struct Array { - GenericValue* elements; - SizeType size; - SizeType capacity; - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - union Data { - String s; - ShortString ss; - Number n; - Object o; - Array a; - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - // Initialize this value as array with initial data, without calling destructor. - void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) { - flags_ = kArrayFlag; - data_.a.elements = (GenericValue*)allocator.Malloc(count * sizeof(GenericValue)); - std::memcpy(data_.a.elements, values, count * sizeof(GenericValue)); - data_.a.size = data_.a.capacity = count; - } - - //! Initialize this value as object with initial data, without calling destructor. - void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) { - flags_ = kObjectFlag; - data_.o.members = (Member*)allocator.Malloc(count * sizeof(Member)); - std::memcpy(data_.o.members, members, count * sizeof(Member)); - data_.o.size = data_.o.capacity = count; - } - - //! Initialize this value as constant string, without calling destructor. - void SetStringRaw(StringRefType s) RAPIDJSON_NOEXCEPT { - flags_ = kConstStringFlag; - data_.s.str = s; - data_.s.length = s.length; - } - - //! Initialize this value as copy string with initial data, without calling destructor. - void SetStringRaw(StringRefType s, Allocator& allocator) { - Ch* str = NULL; - if(ShortString::Usable(s.length)) { - flags_ = kShortStringFlag; - data_.ss.SetLength(s.length); - str = data_.ss.str; - } else { - flags_ = kCopyStringFlag; - data_.s.length = s.length; - str = (Ch *)allocator.Malloc((s.length + 1) * sizeof(Ch)); - data_.s.str = str; - } - std::memcpy(str, s, s.length * sizeof(Ch)); - str[s.length] = '\0'; - } - - //! Assignment without calling destructor - void RawAssign(GenericValue& rhs) RAPIDJSON_NOEXCEPT { - data_ = rhs.data_; - flags_ = rhs.flags_; - rhs.flags_ = kNullFlag; - } - - template - bool StringEqual(const GenericValue& rhs) const { - RAPIDJSON_ASSERT(IsString()); - RAPIDJSON_ASSERT(rhs.IsString()); - - const SizeType len1 = GetStringLength(); - const SizeType len2 = rhs.GetStringLength(); - if(len1 != len2) { return false; } - - const Ch* const str1 = GetString(); - const Ch* const str2 = rhs.GetString(); - if(str1 == str2) { return true; } // fast path for constant string - - return (std::memcmp(str1, str2, sizeof(Ch) * len1) == 0); - } - - Data data_; - unsigned flags_; -}; - -//! GenericValue with UTF8 encoding -typedef GenericValue > Value; - -/////////////////////////////////////////////////////////////////////////////// -// GenericDocument - -//! A document for parsing JSON text as DOM. -/*! - \note implements Handler concept - \tparam Encoding Encoding for both parsing and string storage. - \tparam Allocator Allocator for allocating memory for the DOM - \tparam StackAllocator Allocator for allocating memory for stack during parsing. - \warning Although GenericDocument inherits from GenericValue, the API does \b not provide any virtual functions, especially no virtual destructor. To avoid memory leaks, do not \c delete a GenericDocument object via a pointer to a GenericValue. -*/ -template , typename StackAllocator = CrtAllocator> -class GenericDocument : public GenericValue { -public: - typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. - typedef GenericValue ValueType; //!< Value type of the document. - typedef Allocator AllocatorType; //!< Allocator type from template parameter. - - //! Constructor - /*! \param allocator Optional allocator for allocating memory. - \param stackCapacity Optional initial capacity of stack in bytes. - \param stackAllocator Optional allocator for allocating memory for stack. - */ - GenericDocument(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : - allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() - { - if (!allocator_) - ownAllocator_ = allocator_ = new Allocator(); - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move constructor in C++11 - GenericDocument(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT - : ValueType(std::move(rhs)), - allocator_(rhs.allocator_), - ownAllocator_(rhs.ownAllocator_), - stack_(std::move(rhs.stack_)), - parseResult_(rhs.parseResult_) - { - rhs.allocator_ = 0; - rhs.ownAllocator_ = 0; - rhs.parseResult_ = ParseResult(); - } -#endif - - ~GenericDocument() { - Destroy(); - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move assignment in C++11 - GenericDocument& operator=(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT - { - // The cast to ValueType is necessary here, because otherwise it would - // attempt to call GenericValue's templated assignment operator. - ValueType::operator=(std::forward(rhs)); - - // Calling the destructor here would prematurely call stack_'s destructor - Destroy(); - - allocator_ = rhs.allocator_; - ownAllocator_ = rhs.ownAllocator_; - stack_ = std::move(rhs.stack_); - parseResult_ = rhs.parseResult_; - - rhs.allocator_ = 0; - rhs.ownAllocator_ = 0; - rhs.parseResult_ = ParseResult(); - - return *this; - } -#endif - - //!@name Parse from stream - //!@{ - - //! Parse JSON text from an input stream (with Encoding conversion) - /*! \tparam parseFlags Combination of \ref ParseFlag. - \tparam SourceEncoding Encoding of input stream - \tparam InputStream Type of input stream, implementing Stream concept - \param is Input stream to be parsed. - \return The document itself for fluent API. - */ - template - GenericDocument& ParseStream(InputStream& is) { - ValueType::SetNull(); // Remove existing root if exist - GenericReader reader(&GetAllocator()); - ClearStackOnExit scope(*this); - parseResult_ = reader.template Parse(is, *this); - if (parseResult_) { - RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object - this->RawAssign(*stack_.template Pop(1)); // Add this-> to prevent issue 13. - } - return *this; - } - - //! Parse JSON text from an input stream - /*! \tparam parseFlags Combination of \ref ParseFlag. - \tparam InputStream Type of input stream, implementing Stream concept - \param is Input stream to be parsed. - \return The document itself for fluent API. - */ - template - GenericDocument& ParseStream(InputStream& is) { - return ParseStream(is); - } - - //! Parse JSON text from an input stream (with \ref kParseDefaultFlags) - /*! \tparam InputStream Type of input stream, implementing Stream concept - \param is Input stream to be parsed. - \return The document itself for fluent API. - */ - template - GenericDocument& ParseStream(InputStream& is) { - return ParseStream(is); - } - //!@} - - //!@name Parse in-place from mutable string - //!@{ - - //! Parse JSON text from a mutable string (with Encoding conversion) - /*! \tparam parseFlags Combination of \ref ParseFlag. - \tparam SourceEncoding Transcoding from input Encoding - \param str Mutable zero-terminated string to be parsed. - \return The document itself for fluent API. - */ - template - GenericDocument& ParseInsitu(Ch* str) { - GenericInsituStringStream s(str); - return ParseStream(s); - } - - //! Parse JSON text from a mutable string - /*! \tparam parseFlags Combination of \ref ParseFlag. - \param str Mutable zero-terminated string to be parsed. - \return The document itself for fluent API. - */ - template - GenericDocument& ParseInsitu(Ch* str) { - return ParseInsitu(str); - } - - //! Parse JSON text from a mutable string (with \ref kParseDefaultFlags) - /*! \param str Mutable zero-terminated string to be parsed. - \return The document itself for fluent API. - */ - GenericDocument& ParseInsitu(Ch* str) { - return ParseInsitu(str); - } - //!@} - - //!@name Parse from read-only string - //!@{ - - //! Parse JSON text from a read-only string (with Encoding conversion) - /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). - \tparam SourceEncoding Transcoding from input Encoding - \param str Read-only zero-terminated string to be parsed. - */ - template - GenericDocument& Parse(const Ch* str) { - RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); - GenericStringStream s(str); - return ParseStream(s); - } - - //! Parse JSON text from a read-only string - /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). - \param str Read-only zero-terminated string to be parsed. - */ - template - GenericDocument& Parse(const Ch* str) { - return Parse(str); - } - - //! Parse JSON text from a read-only string (with \ref kParseDefaultFlags) - /*! \param str Read-only zero-terminated string to be parsed. - */ - GenericDocument& Parse(const Ch* str) { - return Parse(str); - } - //!@} - - //!@name Handling parse errors - //!@{ - - //! Whether a parse error has occured in the last parsing. - bool HasParseError() const { return parseResult_.IsError(); } - - //! Get the \ref ParseErrorCode of last parsing. - ParseErrorCode GetParseError() const { return parseResult_.Code(); } - - //! Get the position of last parsing error in input, 0 otherwise. - size_t GetErrorOffset() const { return parseResult_.Offset(); } - - //!@} - - //! Get the allocator of this document. - Allocator& GetAllocator() { return *allocator_; } - - //! Get the capacity of stack in bytes. - size_t GetStackCapacity() const { return stack_.GetCapacity(); } - -private: - // clear stack on any exit from ParseStream, e.g. due to exception - struct ClearStackOnExit { - explicit ClearStackOnExit(GenericDocument& d) : d_(d) {} - ~ClearStackOnExit() { d_.ClearStack(); } - private: - ClearStackOnExit(const ClearStackOnExit&); - ClearStackOnExit& operator=(const ClearStackOnExit&); - GenericDocument& d_; - }; - - // callers of the following private Handler functions - template friend class GenericReader; // for parsing - template friend class GenericValue; // for deep copying - - // Implementation of Handler - bool Null() { new (stack_.template Push()) ValueType(); return true; } - bool Bool(bool b) { new (stack_.template Push()) ValueType(b); return true; } - bool Int(int i) { new (stack_.template Push()) ValueType(i); return true; } - bool Uint(unsigned i) { new (stack_.template Push()) ValueType(i); return true; } - bool Int64(int64_t i) { new (stack_.template Push()) ValueType(i); return true; } - bool Uint64(uint64_t i) { new (stack_.template Push()) ValueType(i); return true; } - bool Double(double d) { new (stack_.template Push()) ValueType(d); return true; } - - bool String(const Ch* str, SizeType length, bool copy) { - if (copy) - new (stack_.template Push()) ValueType(str, length, GetAllocator()); - else - new (stack_.template Push()) ValueType(str, length); - return true; - } - - bool StartObject() { new (stack_.template Push()) ValueType(kObjectType); return true; } - - bool Key(const Ch* str, SizeType length, bool copy) { return String(str, length, copy); } - - bool EndObject(SizeType memberCount) { - typename ValueType::Member* members = stack_.template Pop(memberCount); - stack_.template Top()->SetObjectRaw(members, (SizeType)memberCount, GetAllocator()); - return true; - } - - bool StartArray() { new (stack_.template Push()) ValueType(kArrayType); return true; } - - bool EndArray(SizeType elementCount) { - ValueType* elements = stack_.template Pop(elementCount); - stack_.template Top()->SetArrayRaw(elements, elementCount, GetAllocator()); - return true; - } - -private: - //! Prohibit assignment - GenericDocument& operator=(const GenericDocument&); - - void ClearStack() { - if (Allocator::kNeedFree) - while (stack_.GetSize() > 0) // Here assumes all elements in stack array are GenericValue (Member is actually 2 GenericValue objects) - (stack_.template Pop(1))->~ValueType(); - else - stack_.Clear(); - stack_.ShrinkToFit(); - } - - void Destroy() { - delete ownAllocator_; - } - - static const size_t kDefaultStackCapacity = 1024; - Allocator* allocator_; - Allocator* ownAllocator_; - internal::Stack stack_; - ParseResult parseResult_; -}; - -//! GenericDocument with UTF8 encoding -typedef GenericDocument > Document; - -// defined here due to the dependency on GenericDocument -template -template -inline -GenericValue::GenericValue(const GenericValue& rhs, Allocator& allocator) -{ - GenericDocument d(&allocator); - rhs.Accept(d); - RawAssign(*d.stack_.template Pop(1)); -} - -} // namespace rapidjson - -#if defined(_MSC_VER) || defined(__GNUC__) -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_DOCUMENT_H_ diff --git a/tools/optimizer/rapidjson/encodedstream.h b/tools/optimizer/rapidjson/encodedstream.h deleted file mode 100644 index 9dc00c7afc32c..0000000000000 --- a/tools/optimizer/rapidjson/encodedstream.h +++ /dev/null @@ -1,290 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_ENCODEDSTREAM_H_ -#define RAPIDJSON_ENCODEDSTREAM_H_ - -#include "rapidjson.h" - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#endif - -namespace rapidjson { - -//! Input byte stream wrapper with a statically bound encoding. -/*! - \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. - \tparam InputByteStream Type of input byte stream. For example, FileReadStream. -*/ -template -class EncodedInputStream { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); -public: - typedef typename Encoding::Ch Ch; - - EncodedInputStream(InputByteStream& is) : is_(is) { - current_ = Encoding::TakeBOM(is_); - } - - Ch Peek() const { return current_; } - Ch Take() { Ch c = current_; current_ = Encoding::Take(is_); return c; } - size_t Tell() const { return is_.Tell(); } - - // Not implemented - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - -private: - EncodedInputStream(const EncodedInputStream&); - EncodedInputStream& operator=(const EncodedInputStream&); - - InputByteStream& is_; - Ch current_; -}; - -//! Output byte stream wrapper with statically bound encoding. -/*! - \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. - \tparam InputByteStream Type of input byte stream. For example, FileWriteStream. -*/ -template -class EncodedOutputStream { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); -public: - typedef typename Encoding::Ch Ch; - - EncodedOutputStream(OutputByteStream& os, bool putBOM = true) : os_(os) { - if (putBOM) - Encoding::PutBOM(os_); - } - - void Put(Ch c) { Encoding::Put(os_, c); } - void Flush() { os_.Flush(); } - - // Not implemented - Ch Peek() const { RAPIDJSON_ASSERT(false); } - Ch Take() { RAPIDJSON_ASSERT(false); } - size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - -private: - EncodedOutputStream(const EncodedOutputStream&); - EncodedOutputStream& operator=(const EncodedOutputStream&); - - OutputByteStream& os_; -}; - -#define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x - -//! Input stream wrapper with dynamically bound encoding and automatic encoding detection. -/*! - \tparam CharType Type of character for reading. - \tparam InputByteStream type of input byte stream to be wrapped. -*/ -template -class AutoUTFInputStream { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); -public: - typedef CharType Ch; - - //! Constructor. - /*! - \param is input stream to be wrapped. - \param type UTF encoding type if it is not detected from the stream. - */ - AutoUTFInputStream(InputByteStream& is, UTFType type = kUTF8) : is_(&is), type_(type), hasBOM_(false) { - DetectType(); - static const TakeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Take) }; - takeFunc_ = f[type_]; - current_ = takeFunc_(*is_); - } - - UTFType GetType() const { return type_; } - bool HasBOM() const { return hasBOM_; } - - Ch Peek() const { return current_; } - Ch Take() { Ch c = current_; current_ = takeFunc_(*is_); return c; } - size_t Tell() const { return is_->Tell(); } - - // Not implemented - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - -private: - AutoUTFInputStream(const AutoUTFInputStream&); - AutoUTFInputStream& operator=(const AutoUTFInputStream&); - - // Detect encoding type with BOM or RFC 4627 - void DetectType() { - // BOM (Byte Order Mark): - // 00 00 FE FF UTF-32BE - // FF FE 00 00 UTF-32LE - // FE FF UTF-16BE - // FF FE UTF-16LE - // EF BB BF UTF-8 - - const unsigned char* c = (const unsigned char *)is_->Peek4(); - if (!c) - return; - - unsigned bom = c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24); - hasBOM_ = false; - if (bom == 0xFFFE0000) { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } - else if (bom == 0x0000FEFF) { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } - else if ((bom & 0xFFFF) == 0xFFFE) { type_ = kUTF16BE; hasBOM_ = true; is_->Take(); is_->Take(); } - else if ((bom & 0xFFFF) == 0xFEFF) { type_ = kUTF16LE; hasBOM_ = true; is_->Take(); is_->Take(); } - else if ((bom & 0xFFFFFF) == 0xBFBBEF) { type_ = kUTF8; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); } - - // RFC 4627: Section 3 - // "Since the first two characters of a JSON text will always be ASCII - // characters [RFC0020], it is possible to determine whether an octet - // stream is UTF-8, UTF-16 (BE or LE), or UTF-32 (BE or LE) by looking - // at the pattern of nulls in the first four octets." - // 00 00 00 xx UTF-32BE - // 00 xx 00 xx UTF-16BE - // xx 00 00 00 UTF-32LE - // xx 00 xx 00 UTF-16LE - // xx xx xx xx UTF-8 - - if (!hasBOM_) { - unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); - switch (pattern) { - case 0x08: type_ = kUTF32BE; break; - case 0x0A: type_ = kUTF16BE; break; - case 0x01: type_ = kUTF32LE; break; - case 0x05: type_ = kUTF16LE; break; - case 0x0F: type_ = kUTF8; break; - default: break; // Use type defined by user. - } - } - - // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. - switch (type_) { - case kUTF8: - // Do nothing - break; - case kUTF16LE: - case kUTF16BE: - RAPIDJSON_ASSERT(sizeof(Ch) >= 2); - break; - case kUTF32LE: - case kUTF32BE: - RAPIDJSON_ASSERT(sizeof(Ch) >= 4); - break; - default: - RAPIDJSON_ASSERT(false); // Invalid type - } - } - - typedef Ch (*TakeFunc)(InputByteStream& is); - InputByteStream* is_; - UTFType type_; - Ch current_; - TakeFunc takeFunc_; - bool hasBOM_; -}; - -//! Output stream wrapper with dynamically bound encoding and automatic encoding detection. -/*! - \tparam CharType Type of character for writing. - \tparam InputByteStream type of output byte stream to be wrapped. -*/ -template -class AutoUTFOutputStream { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); -public: - typedef CharType Ch; - - //! Constructor. - /*! - \param os output stream to be wrapped. - \param type UTF encoding type. - \param putBOM Whether to write BOM at the beginning of the stream. - */ - AutoUTFOutputStream(OutputByteStream& os, UTFType type, bool putBOM) : os_(&os), type_(type) { - // RUntime check whether the size of character type is sufficient. It only perform checks with assertion. - switch (type_) { - case kUTF16LE: - case kUTF16BE: - RAPIDJSON_ASSERT(sizeof(Ch) >= 2); - break; - case kUTF32LE: - case kUTF32BE: - RAPIDJSON_ASSERT(sizeof(Ch) >= 4); - break; - case kUTF8: - // Do nothing - break; - default: - RAPIDJSON_ASSERT(false); // Invalid UTFType - } - - static const PutFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Put) }; - putFunc_ = f[type_]; - - if (putBOM) - PutBOM(); - } - - UTFType GetType() const { return type_; } - - void Put(Ch c) { putFunc_(*os_, c); } - void Flush() { os_->Flush(); } - - // Not implemented - Ch Peek() const { RAPIDJSON_ASSERT(false); } - Ch Take() { RAPIDJSON_ASSERT(false); } - size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - -private: - AutoUTFOutputStream(const AutoUTFOutputStream&); - AutoUTFOutputStream& operator=(const AutoUTFOutputStream&); - - void PutBOM() { - typedef void (*PutBOMFunc)(OutputByteStream&); - static const PutBOMFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(PutBOM) }; - f[type_](*os_); - } - - typedef void (*PutFunc)(OutputByteStream&, Ch); - - OutputByteStream* os_; - UTFType type_; - PutFunc putFunc_; -}; - -#undef RAPIDJSON_ENCODINGS_FUNC - -} // namespace rapidjson - -#ifdef __GNUC__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/tools/optimizer/rapidjson/encodings.h b/tools/optimizer/rapidjson/encodings.h deleted file mode 100644 index 5c7da3e048c69..0000000000000 --- a/tools/optimizer/rapidjson/encodings.h +++ /dev/null @@ -1,630 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_ENCODINGS_H_ -#define RAPIDJSON_ENCODINGS_H_ - -#include "rapidjson.h" - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4244) // conversion from 'type1' to 'type2', possible loss of data -RAPIDJSON_DIAG_OFF(4702) // unreachable code -#elif defined(__GNUC__) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#endif - -namespace rapidjson { - -/////////////////////////////////////////////////////////////////////////////// -// Encoding - -/*! \class rapidjson::Encoding - \brief Concept for encoding of Unicode characters. - -\code -concept Encoding { - typename Ch; //! Type of character. A "character" is actually a code unit in unicode's definition. - - enum { supportUnicode = 1 }; // or 0 if not supporting unicode - - //! \brief Encode a Unicode codepoint to an output stream. - //! \param os Output stream. - //! \param codepoint An unicode codepoint, ranging from 0x0 to 0x10FFFF inclusively. - template - static void Encode(OutputStream& os, unsigned codepoint); - - //! \brief Decode a Unicode codepoint from an input stream. - //! \param is Input stream. - //! \param codepoint Output of the unicode codepoint. - //! \return true if a valid codepoint can be decoded from the stream. - template - static bool Decode(InputStream& is, unsigned* codepoint); - - //! \brief Validate one Unicode codepoint from an encoded stream. - //! \param is Input stream to obtain codepoint. - //! \param os Output for copying one codepoint. - //! \return true if it is valid. - //! \note This function just validating and copying the codepoint without actually decode it. - template - static bool Validate(InputStream& is, OutputStream& os); - - // The following functions are deal with byte streams. - - //! Take a character from input byte stream, skip BOM if exist. - template - static CharType TakeBOM(InputByteStream& is); - - //! Take a character from input byte stream. - template - static Ch Take(InputByteStream& is); - - //! Put BOM to output byte stream. - template - static void PutBOM(OutputByteStream& os); - - //! Put a character to output byte stream. - template - static void Put(OutputByteStream& os, Ch c); -}; -\endcode -*/ - -/////////////////////////////////////////////////////////////////////////////// -// UTF8 - -//! UTF-8 encoding. -/*! http://en.wikipedia.org/wiki/UTF-8 - http://tools.ietf.org/html/rfc3629 - \tparam CharType Code unit for storing 8-bit UTF-8 data. Default is char. - \note implements Encoding concept -*/ -template -struct UTF8 { - typedef CharType Ch; - - enum { supportUnicode = 1 }; - - template - static void Encode(OutputStream& os, unsigned codepoint) { - if (codepoint <= 0x7F) - os.Put(static_cast(codepoint & 0xFF)); - else if (codepoint <= 0x7FF) { - os.Put(static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); - os.Put(static_cast(0x80 | ((codepoint & 0x3F)))); - } - else if (codepoint <= 0xFFFF) { - os.Put(static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); - os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); - os.Put(static_cast(0x80 | (codepoint & 0x3F))); - } - else { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - os.Put(static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); - os.Put(static_cast(0x80 | ((codepoint >> 12) & 0x3F))); - os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); - os.Put(static_cast(0x80 | (codepoint & 0x3F))); - } - } - - template - static bool Decode(InputStream& is, unsigned* codepoint) { -#define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | ((unsigned char)c & 0x3Fu) -#define TRANS(mask) result &= ((GetRange((unsigned char)c) & mask) != 0) -#define TAIL() COPY(); TRANS(0x70) - Ch c = is.Take(); - if (!(c & 0x80)) { - *codepoint = (unsigned char)c; - return true; - } - - unsigned char type = GetRange((unsigned char)c); - *codepoint = (0xFF >> type) & (unsigned char)c; - bool result = true; - switch (type) { - case 2: TAIL(); return result; - case 3: TAIL(); TAIL(); return result; - case 4: COPY(); TRANS(0x50); TAIL(); return result; - case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; - case 6: TAIL(); TAIL(); TAIL(); return result; - case 10: COPY(); TRANS(0x20); TAIL(); return result; - case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; - default: return false; - } -#undef COPY -#undef TRANS -#undef TAIL - } - - template - static bool Validate(InputStream& is, OutputStream& os) { -#define COPY() os.Put(c = is.Take()) -#define TRANS(mask) result &= ((GetRange((unsigned char)c) & mask) != 0) -#define TAIL() COPY(); TRANS(0x70) - Ch c; - COPY(); - if (!(c & 0x80)) - return true; - - bool result = true; - switch (GetRange((unsigned char)c)) { - case 2: TAIL(); return result; - case 3: TAIL(); TAIL(); return result; - case 4: COPY(); TRANS(0x50); TAIL(); return result; - case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; - case 6: TAIL(); TAIL(); TAIL(); return result; - case 10: COPY(); TRANS(0x20); TAIL(); return result; - case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; - default: return false; - } -#undef COPY -#undef TRANS -#undef TAIL - } - - static unsigned char GetRange(unsigned char c) { - // Referring to DFA of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ - // With new mapping 1 -> 0x10, 7 -> 0x20, 9 -> 0x40, such that AND operation can test multiple types. - static const unsigned char type[] = { - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, - 0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40, - 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, - 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, - 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, - }; - return type[c]; - } - - template - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - Ch c = Take(is); - if ((unsigned char)c != 0xEFu) return c; - c = is.Take(); - if ((unsigned char)c != 0xBBu) return c; - c = is.Take(); - if ((unsigned char)c != 0xBFu) return c; - c = is.Take(); - return c; - } - - template - static Ch Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - return is.Take(); - } - - template - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0xEFu); os.Put(0xBBu); os.Put(0xBFu); - } - - template - static void Put(OutputByteStream& os, Ch c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(c)); - } -}; - -/////////////////////////////////////////////////////////////////////////////// -// UTF16 - -//! UTF-16 encoding. -/*! http://en.wikipedia.org/wiki/UTF-16 - http://tools.ietf.org/html/rfc2781 - \tparam CharType Type for storing 16-bit UTF-16 data. Default is wchar_t. C++11 may use char16_t instead. - \note implements Encoding concept - - \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. - For streaming, use UTF16LE and UTF16BE, which handle endianness. -*/ -template -struct UTF16 { - typedef CharType Ch; - RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 2); - - enum { supportUnicode = 1 }; - - template - static void Encode(OutputStream& os, unsigned codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); - if (codepoint <= 0xFFFF) { - RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair - os.Put(static_cast(codepoint)); - } - else { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - unsigned v = codepoint - 0x10000; - os.Put(static_cast((v >> 10) | 0xD800)); - os.Put((v & 0x3FF) | 0xDC00); - } - } - - template - static bool Decode(InputStream& is, unsigned* codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); - Ch c = is.Take(); - if (c < 0xD800 || c > 0xDFFF) { - *codepoint = c; - return true; - } - else if (c <= 0xDBFF) { - *codepoint = (c & 0x3FF) << 10; - c = is.Take(); - *codepoint |= (c & 0x3FF); - *codepoint += 0x10000; - return c >= 0xDC00 && c <= 0xDFFF; - } - return false; - } - - template - static bool Validate(InputStream& is, OutputStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); - Ch c; - os.Put(c = is.Take()); - if (c < 0xD800 || c > 0xDFFF) - return true; - else if (c <= 0xDBFF) { - os.Put(c = is.Take()); - return c >= 0xDC00 && c <= 0xDFFF; - } - return false; - } -}; - -//! UTF-16 little endian encoding. -template -struct UTF16LE : UTF16 { - template - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = Take(is); - return (unsigned short)c == 0xFEFFu ? Take(is) : c; - } - - template - static CharType Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = (unsigned char)is.Take(); - c |= (unsigned char)is.Take() << 8; - return c; - } - - template - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0xFFu); os.Put(0xFEu); - } - - template - static void Put(OutputByteStream& os, CharType c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(c & 0xFFu); - os.Put((c >> 8) & 0xFFu); - } -}; - -//! UTF-16 big endian encoding. -template -struct UTF16BE : UTF16 { - template - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = Take(is); - return (unsigned short)c == 0xFEFFu ? Take(is) : c; - } - - template - static CharType Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = (unsigned char)is.Take() << 8; - c |= (unsigned char)is.Take(); - return c; - } - - template - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0xFEu); os.Put(0xFFu); - } - - template - static void Put(OutputByteStream& os, CharType c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put((c >> 8) & 0xFFu); - os.Put(c & 0xFFu); - } -}; - -/////////////////////////////////////////////////////////////////////////////// -// UTF32 - -//! UTF-32 encoding. -/*! http://en.wikipedia.org/wiki/UTF-32 - \tparam CharType Type for storing 32-bit UTF-32 data. Default is unsigned. C++11 may use char32_t instead. - \note implements Encoding concept - - \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. - For streaming, use UTF32LE and UTF32BE, which handle endianness. -*/ -template -struct UTF32 { - typedef CharType Ch; - RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 4); - - enum { supportUnicode = 1 }; - - template - static void Encode(OutputStream& os, unsigned codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - os.Put(codepoint); - } - - template - static bool Decode(InputStream& is, unsigned* codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); - Ch c = is.Take(); - *codepoint = c; - return c <= 0x10FFFF; - } - - template - static bool Validate(InputStream& is, OutputStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); - Ch c; - os.Put(c = is.Take()); - return c <= 0x10FFFF; - } -}; - -//! UTF-32 little endian enocoding. -template -struct UTF32LE : UTF32 { - template - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = Take(is); - return (unsigned)c == 0x0000FEFFu ? Take(is) : c; - } - - template - static CharType Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = (unsigned char)is.Take(); - c |= (unsigned char)is.Take() << 8; - c |= (unsigned char)is.Take() << 16; - c |= (unsigned char)is.Take() << 24; - return c; - } - - template - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0xFFu); os.Put(0xFEu); os.Put(0x00u); os.Put(0x00u); - } - - template - static void Put(OutputByteStream& os, CharType c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(c & 0xFFu); - os.Put((c >> 8) & 0xFFu); - os.Put((c >> 16) & 0xFFu); - os.Put((c >> 24) & 0xFFu); - } -}; - -//! UTF-32 big endian encoding. -template -struct UTF32BE : UTF32 { - template - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = Take(is); - return (unsigned)c == 0x0000FEFFu ? Take(is) : c; - } - - template - static CharType Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = (unsigned char)is.Take() << 24; - c |= (unsigned char)is.Take() << 16; - c |= (unsigned char)is.Take() << 8; - c |= (unsigned char)is.Take(); - return c; - } - - template - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0x00u); os.Put(0x00u); os.Put(0xFEu); os.Put(0xFFu); - } - - template - static void Put(OutputByteStream& os, CharType c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put((c >> 24) & 0xFFu); - os.Put((c >> 16) & 0xFFu); - os.Put((c >> 8) & 0xFFu); - os.Put(c & 0xFFu); - } -}; - -/////////////////////////////////////////////////////////////////////////////// -// ASCII - -//! ASCII encoding. -/*! http://en.wikipedia.org/wiki/ASCII - \tparam CharType Code unit for storing 7-bit ASCII data. Default is char. - \note implements Encoding concept -*/ -template -struct ASCII { - typedef CharType Ch; - - enum { supportUnicode = 0 }; - - template - static void Encode(OutputStream& os, unsigned codepoint) { - RAPIDJSON_ASSERT(codepoint <= 0x7F); - os.Put(static_cast(codepoint & 0xFF)); - } - - template - static bool Decode(InputStream& is, unsigned* codepoint) { - unsigned char c = static_cast(is.Take()); - *codepoint = c; - return c <= 0X7F; - } - - template - static bool Validate(InputStream& is, OutputStream& os) { - unsigned char c = is.Take(); - os.Put(c); - return c <= 0x7F; - } - - template - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - Ch c = Take(is); - return c; - } - - template - static Ch Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - return is.Take(); - } - - template - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - (void)os; - } - - template - static void Put(OutputByteStream& os, Ch c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(c)); - } -}; - -/////////////////////////////////////////////////////////////////////////////// -// AutoUTF - -//! Runtime-specified UTF encoding type of a stream. -enum UTFType { - kUTF8 = 0, //!< UTF-8. - kUTF16LE = 1, //!< UTF-16 little endian. - kUTF16BE = 2, //!< UTF-16 big endian. - kUTF32LE = 3, //!< UTF-32 little endian. - kUTF32BE = 4 //!< UTF-32 big endian. -}; - -//! Dynamically select encoding according to stream's runtime-specified UTF encoding type. -/*! \note This class can be used with AutoUTFInputtStream and AutoUTFOutputStream, which provides GetType(). -*/ -template -struct AutoUTF { - typedef CharType Ch; - - enum { supportUnicode = 1 }; - -#define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x - - template - RAPIDJSON_FORCEINLINE static void Encode(OutputStream& os, unsigned codepoint) { - typedef void (*EncodeFunc)(OutputStream&, unsigned); - static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Encode) }; - (*f[os.GetType()])(os, codepoint); - } - - template - RAPIDJSON_FORCEINLINE static bool Decode(InputStream& is, unsigned* codepoint) { - typedef bool (*DecodeFunc)(InputStream&, unsigned*); - static const DecodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Decode) }; - return (*f[is.GetType()])(is, codepoint); - } - - template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { - typedef bool (*ValidateFunc)(InputStream&, OutputStream&); - static const ValidateFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Validate) }; - return (*f[is.GetType()])(is, os); - } - -#undef RAPIDJSON_ENCODINGS_FUNC -}; - -/////////////////////////////////////////////////////////////////////////////// -// Transcoder - -//! Encoding conversion. -template -struct Transcoder { - //! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream. - template - RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { - unsigned codepoint; - if (!SourceEncoding::Decode(is, &codepoint)) - return false; - TargetEncoding::Encode(os, codepoint); - return true; - } - - //! Validate one Unicode codepoint from an encoded stream. - template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { - return Transcode(is, os); // Since source/target encoding is different, must transcode. - } -}; - -//! Specialization of Transcoder with same source and target encoding. -template -struct Transcoder { - template - RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { - os.Put(is.Take()); // Just copy one code unit. This semantic is different from primary template class. - return true; - } - - template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { - return Encoding::Validate(is, os); // source/target encoding are the same - } -}; - -} // namespace rapidjson - -#if defined(__GNUC__) || defined(_MSV_VER) -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_ENCODINGS_H_ diff --git a/tools/optimizer/rapidjson/error/en.h b/tools/optimizer/rapidjson/error/en.h deleted file mode 100644 index d153e04fc8411..0000000000000 --- a/tools/optimizer/rapidjson/error/en.h +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_ERROR_EN_H__ -#define RAPIDJSON_ERROR_EN_H__ - -#include "error.h" - -namespace rapidjson { - -//! Maps error code of parsing into error message. -/*! - \ingroup RAPIDJSON_ERRORS - \param parseErrorCode Error code obtained in parsing. - \return the error message. - \note User can make a copy of this function for localization. - Using switch-case is safer for future modification of error codes. -*/ -inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErrorCode) { - switch (parseErrorCode) { - case kParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); - - case kParseErrorDocumentEmpty: return RAPIDJSON_ERROR_STRING("The document is empty."); - case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not follow by other values."); - - case kParseErrorValueInvalid: return RAPIDJSON_ERROR_STRING("Invalid value."); - - case kParseErrorObjectMissName: return RAPIDJSON_ERROR_STRING("Missing a name for object member."); - case kParseErrorObjectMissColon: return RAPIDJSON_ERROR_STRING("Missing a colon after a name of object member."); - case kParseErrorObjectMissCommaOrCurlyBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or '}' after an object member."); - - case kParseErrorArrayMissCommaOrSquareBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or ']' after an array element."); - - case kParseErrorStringUnicodeEscapeInvalidHex: return RAPIDJSON_ERROR_STRING("Incorrect hex digit after \\u escape in string."); - case kParseErrorStringUnicodeSurrogateInvalid: return RAPIDJSON_ERROR_STRING("The surrogate pair in string is invalid."); - case kParseErrorStringEscapeInvalid: return RAPIDJSON_ERROR_STRING("Invalid escape character in string."); - case kParseErrorStringMissQuotationMark: return RAPIDJSON_ERROR_STRING("Missing a closing quotation mark in string."); - case kParseErrorStringInvalidEncoding: return RAPIDJSON_ERROR_STRING("Invalid encoding in string."); - - case kParseErrorNumberTooBig: return RAPIDJSON_ERROR_STRING("Number too big to be stored in double."); - case kParseErrorNumberMissFraction: return RAPIDJSON_ERROR_STRING("Miss fraction part in number."); - case kParseErrorNumberMissExponent: return RAPIDJSON_ERROR_STRING("Miss exponent in number."); - - case kParseErrorTermination: return RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error."); - case kParseErrorUnspecificSyntaxError: return RAPIDJSON_ERROR_STRING("Unspecific syntax error."); - - default: - return RAPIDJSON_ERROR_STRING("Unknown error."); - } -} - -} // namespace rapidjson - -#endif // RAPIDJSON_ERROR_EN_H__ diff --git a/tools/optimizer/rapidjson/error/error.h b/tools/optimizer/rapidjson/error/error.h deleted file mode 100644 index 146604439b16a..0000000000000 --- a/tools/optimizer/rapidjson/error/error.h +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_ERROR_ERROR_H__ -#define RAPIDJSON_ERROR_ERROR_H__ - -/*! \file error.h */ - -/*! \defgroup RAPIDJSON_ERRORS RapidJSON error handling */ - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_ERROR_CHARTYPE - -//! Character type of error messages. -/*! \ingroup RAPIDJSON_ERRORS - The default character type is \c char. - On Windows, user can define this macro as \c TCHAR for supporting both - unicode/non-unicode settings. -*/ -#ifndef RAPIDJSON_ERROR_CHARTYPE -#define RAPIDJSON_ERROR_CHARTYPE char -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_ERROR_STRING - -//! Macro for converting string literial to \ref RAPIDJSON_ERROR_CHARTYPE[]. -/*! \ingroup RAPIDJSON_ERRORS - By default this conversion macro does nothing. - On Windows, user can define this macro as \c _T(x) for supporting both - unicode/non-unicode settings. -*/ -#ifndef RAPIDJSON_ERROR_STRING -#define RAPIDJSON_ERROR_STRING(x) x -#endif - -namespace rapidjson { - -/////////////////////////////////////////////////////////////////////////////// -// ParseErrorCode - -//! Error code of parsing. -/*! \ingroup RAPIDJSON_ERRORS - \see GenericReader::Parse, GenericReader::GetParseErrorCode -*/ -enum ParseErrorCode { - kParseErrorNone = 0, //!< No error. - - kParseErrorDocumentEmpty, //!< The document is empty. - kParseErrorDocumentRootNotSingular, //!< The document root must not follow by other values. - - kParseErrorValueInvalid, //!< Invalid value. - - kParseErrorObjectMissName, //!< Missing a name for object member. - kParseErrorObjectMissColon, //!< Missing a colon after a name of object member. - kParseErrorObjectMissCommaOrCurlyBracket, //!< Missing a comma or '}' after an object member. - - kParseErrorArrayMissCommaOrSquareBracket, //!< Missing a comma or ']' after an array element. - - kParseErrorStringUnicodeEscapeInvalidHex, //!< Incorrect hex digit after \\u escape in string. - kParseErrorStringUnicodeSurrogateInvalid, //!< The surrogate pair in string is invalid. - kParseErrorStringEscapeInvalid, //!< Invalid escape character in string. - kParseErrorStringMissQuotationMark, //!< Missing a closing quotation mark in string. - kParseErrorStringInvalidEncoding, //!< Invalid encoding in string. - - kParseErrorNumberTooBig, //!< Number too big to be stored in double. - kParseErrorNumberMissFraction, //!< Miss fraction part in number. - kParseErrorNumberMissExponent, //!< Miss exponent in number. - - kParseErrorTermination, //!< Parsing was terminated. - kParseErrorUnspecificSyntaxError, //!< Unspecific syntax error. -}; - -//! Result of parsing (wraps ParseErrorCode) -/*! - \ingroup RAPIDJSON_ERRORS - \code - Document doc; - ParseResult ok = doc.Parse("[42]"); - if (!ok) { - fprintf(stderr, "JSON parse error: %s (%u)", - GetParseError_En(ok.Code()), ok.Offset()); - exit(EXIT_FAILURE); - } - \endcode - \see GenericReader::Parse, GenericDocument::Parse -*/ -struct ParseResult { - - //! Default constructor, no error. - ParseResult() : code_(kParseErrorNone), offset_(0) {} - //! Constructor to set an error. - ParseResult(ParseErrorCode code, size_t offset) : code_(code), offset_(offset) {} - - //! Get the error code. - ParseErrorCode Code() const { return code_; } - //! Get the error offset, if \ref IsError(), 0 otherwise. - size_t Offset() const { return offset_; } - - //! Conversion to \c bool, returns \c true, iff !\ref IsError(). - operator bool() const { return !IsError(); } - //! Whether the result is an error. - bool IsError() const { return code_ != kParseErrorNone; } - - bool operator==(const ParseResult& that) const { return code_ == that.code_; } - bool operator==(ParseErrorCode code) const { return code_ == code; } - friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; } - - //! Reset error code. - void Clear() { Set(kParseErrorNone); } - //! Update error code and offset. - void Set(ParseErrorCode code, size_t offset = 0) { code_ = code; offset_ = offset; } - -private: - ParseErrorCode code_; - size_t offset_; -}; - -//! Function pointer type of GetParseError(). -/*! \ingroup RAPIDJSON_ERRORS - - This is the prototype for \c GetParseError_X(), where \c X is a locale. - User can dynamically change locale in runtime, e.g.: -\code - GetParseErrorFunc GetParseError = GetParseError_En; // or whatever - const RAPIDJSON_ERROR_CHARTYPE* s = GetParseError(document.GetParseErrorCode()); -\endcode -*/ -typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetParseErrorFunc)(ParseErrorCode); - -} // namespace rapidjson - -#endif // RAPIDJSON_ERROR_ERROR_H__ diff --git a/tools/optimizer/rapidjson/filereadstream.h b/tools/optimizer/rapidjson/filereadstream.h deleted file mode 100644 index 31c193bf98d07..0000000000000 --- a/tools/optimizer/rapidjson/filereadstream.h +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_FILEREADSTREAM_H_ -#define RAPIDJSON_FILEREADSTREAM_H_ - -#include "rapidjson.h" -#include - -namespace rapidjson { - -//! File byte stream for input using fread(). -/*! - \note implements Stream concept -*/ -class FileReadStream { -public: - typedef char Ch; //!< Character type (byte). - - //! Constructor. - /*! - \param fp File pointer opened for read. - \param buffer user-supplied buffer. - \param bufferSize size of buffer in bytes. Must >=4 bytes. - */ - FileReadStream(FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { - RAPIDJSON_ASSERT(fp_ != 0); - RAPIDJSON_ASSERT(bufferSize >= 4); - Read(); - } - - Ch Peek() const { return *current_; } - Ch Take() { Ch c = *current_; Read(); return c; } - size_t Tell() const { return count_ + static_cast(current_ - buffer_); } - - // Not implemented - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - - // For encoding detection only. - const Ch* Peek4() const { - return (current_ + 4 <= bufferLast_) ? current_ : 0; - } - -private: - void Read() { - if (current_ < bufferLast_) - ++current_; - else if (!eof_) { - count_ += readCount_; - readCount_ = fread(buffer_, 1, bufferSize_, fp_); - bufferLast_ = buffer_ + readCount_ - 1; - current_ = buffer_; - - if (readCount_ < bufferSize_) { - buffer_[readCount_] = '\0'; - ++bufferLast_; - eof_ = true; - } - } - } - - FILE* fp_; - Ch *buffer_; - size_t bufferSize_; - Ch *bufferLast_; - Ch *current_; - size_t readCount_; - size_t count_; //!< Number of characters read - bool eof_; -}; - -} // namespace rapidjson - -#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/tools/optimizer/rapidjson/filestream.h b/tools/optimizer/rapidjson/filestream.h deleted file mode 100644 index 5fe00362b560a..0000000000000 --- a/tools/optimizer/rapidjson/filestream.h +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_FILESTREAM_H_ -#define RAPIDJSON_FILESTREAM_H_ - -#include "rapidjson.h" -#include - -namespace rapidjson { - -//! (Deprecated) Wrapper of C file stream for input or output. -/*! - This simple wrapper does not check the validity of the stream. - \note implements Stream concept - \note deprecated: This was only for basic testing in version 0.1, it is found that the performance is very low by using fgetc(). Use FileReadStream instead. -*/ -class FileStream { -public: - typedef char Ch; //!< Character type. Only support char. - - FileStream(FILE* fp) : fp_(fp), current_('\0'), count_(0) { Read(); } - char Peek() const { return current_; } - char Take() { char c = current_; Read(); return c; } - size_t Tell() const { return count_; } - void Put(char c) { fputc(c, fp_); } - void Flush() { fflush(fp_); } - - // Not implemented - char* PutBegin() { return 0; } - size_t PutEnd(char*) { return 0; } - -private: - // Prohibit copy constructor & assignment operator. - FileStream(const FileStream&); - FileStream& operator=(const FileStream&); - - void Read() { - RAPIDJSON_ASSERT(fp_ != 0); - int c = fgetc(fp_); - if (c != EOF) { - current_ = (char)c; - count_++; - } - else if (current_ != '\0') - current_ = '\0'; - } - - FILE* fp_; - char current_; - size_t count_; -}; - -} // namespace rapidjson - -#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/tools/optimizer/rapidjson/filewritestream.h b/tools/optimizer/rapidjson/filewritestream.h deleted file mode 100644 index 05c5ca09077eb..0000000000000 --- a/tools/optimizer/rapidjson/filewritestream.h +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_FILEWRITESTREAM_H_ -#define RAPIDJSON_FILEWRITESTREAM_H_ - -#include "rapidjson.h" -#include - -namespace rapidjson { - -//! Wrapper of C file stream for input using fread(). -/*! - \note implements Stream concept -*/ -class FileWriteStream { -public: - typedef char Ch; //!< Character type. Only support char. - - FileWriteStream(FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { - RAPIDJSON_ASSERT(fp_ != 0); - } - - void Put(char c) { - if (current_ >= bufferEnd_) - Flush(); - - *current_++ = c; - } - - void PutN(char c, size_t n) { - size_t avail = static_cast(bufferEnd_ - current_); - while (n > avail) { - std::memset(current_, c, avail); - current_ += avail; - Flush(); - n -= avail; - avail = static_cast(bufferEnd_ - current_); - } - - if (n > 0) { - std::memset(current_, c, n); - current_ += n; - } - } - - void Flush() { - if (current_ != buffer_) { - fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); - current_ = buffer_; - } - } - - // Not implemented - char Peek() const { RAPIDJSON_ASSERT(false); return 0; } - char Take() { RAPIDJSON_ASSERT(false); return 0; } - size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } - char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } - -private: - // Prohibit copy constructor & assignment operator. - FileWriteStream(const FileWriteStream&); - FileWriteStream& operator=(const FileWriteStream&); - - FILE* fp_; - char *buffer_; - char *bufferEnd_; - char *current_; -}; - -//! Implement specialized version of PutN() with memset() for better performance. -template<> -inline void PutN(FileWriteStream& stream, char c, size_t n) { - stream.PutN(c, n); -} - -} // namespace rapidjson - -#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/tools/optimizer/rapidjson/internal/dtoa.h b/tools/optimizer/rapidjson/internal/dtoa.h deleted file mode 100644 index 6ae588ac4aa3c..0000000000000 --- a/tools/optimizer/rapidjson/internal/dtoa.h +++ /dev/null @@ -1,418 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// This is a C++ header-only implementation of Grisu2 algorithm from the publication: -// Loitsch, Florian. "Printing floating-point numbers quickly and accurately with -// integers." ACM Sigplan Notices 45.6 (2010): 233-243. - -#ifndef RAPIDJSON_DTOA_ -#define RAPIDJSON_DTOA_ - -#if defined(_MSC_VER) -#include -#if defined(_M_AMD64) -#pragma intrinsic(_BitScanReverse64) -#endif -#endif - -#include "itoa.h" // GetDigitsLut() - -namespace rapidjson { -namespace internal { - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#endif - -struct DiyFp { - DiyFp() {} - - DiyFp(uint64_t f, int e) : f(f), e(e) {} - - DiyFp(double d) { - union { - double d; - uint64_t u64; - } u = { d }; - - int biased_e = static_cast((u.u64 & kDpExponentMask) >> kDpSignificandSize); - uint64_t significand = (u.u64 & kDpSignificandMask); - if (biased_e != 0) { - f = significand + kDpHiddenBit; - e = biased_e - kDpExponentBias; - } - else { - f = significand; - e = kDpMinExponent + 1; - } - } - - DiyFp operator-(const DiyFp& rhs) const { - return DiyFp(f - rhs.f, e); - } - - DiyFp operator*(const DiyFp& rhs) const { -#if defined(_MSC_VER) && defined(_M_AMD64) - uint64_t h; - uint64_t l = _umul128(f, rhs.f, &h); - if (l & (uint64_t(1) << 63)) // rounding - h++; - return DiyFp(h, e + rhs.e + 64); -#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) - unsigned __int128 p = static_cast(f) * static_cast(rhs.f); - uint64_t h = static_cast(p >> 64); - uint64_t l = static_cast(p); - if (l & (uint64_t(1) << 63)) // rounding - h++; - return DiyFp(h, e + rhs.e + 64); -#else - const uint64_t M32 = 0xFFFFFFFF; - const uint64_t a = f >> 32; - const uint64_t b = f & M32; - const uint64_t c = rhs.f >> 32; - const uint64_t d = rhs.f & M32; - const uint64_t ac = a * c; - const uint64_t bc = b * c; - const uint64_t ad = a * d; - const uint64_t bd = b * d; - uint64_t tmp = (bd >> 32) + (ad & M32) + (bc & M32); - tmp += 1U << 31; /// mult_round - return DiyFp(ac + (ad >> 32) + (bc >> 32) + (tmp >> 32), e + rhs.e + 64); -#endif - } - - DiyFp Normalize() const { -#if defined(_MSC_VER) && defined(_M_AMD64) - unsigned long index; - _BitScanReverse64(&index, f); - return DiyFp(f << (63 - index), e - (63 - index)); -#elif defined(__GNUC__) - int s = __builtin_clzll(f); - return DiyFp(f << s, e - s); -#else - DiyFp res = *this; - while (!(res.f & kDpHiddenBit)) { - res.f <<= 1; - res.e--; - } - res.f <<= (kDiySignificandSize - kDpSignificandSize - 1); - res.e = res.e - (kDiySignificandSize - kDpSignificandSize - 1); - return res; -#endif - } - - DiyFp NormalizeBoundary() const { -#if defined(_MSC_VER) && defined(_M_AMD64) - unsigned long index; - _BitScanReverse64(&index, f); - return DiyFp (f << (63 - index), e - (63 - index)); -#else - DiyFp res = *this; - while (!(res.f & (kDpHiddenBit << 1))) { - res.f <<= 1; - res.e--; - } - res.f <<= (kDiySignificandSize - kDpSignificandSize - 2); - res.e = res.e - (kDiySignificandSize - kDpSignificandSize - 2); - return res; -#endif - } - - void NormalizedBoundaries(DiyFp* minus, DiyFp* plus) const { - DiyFp pl = DiyFp((f << 1) + 1, e - 1).NormalizeBoundary(); - DiyFp mi = (f == kDpHiddenBit) ? DiyFp((f << 2) - 1, e - 2) : DiyFp((f << 1) - 1, e - 1); - mi.f <<= mi.e - pl.e; - mi.e = pl.e; - *plus = pl; - *minus = mi; - } - - static const int kDiySignificandSize = 64; - static const int kDpSignificandSize = 52; - static const int kDpExponentBias = 0x3FF + kDpSignificandSize; - static const int kDpMinExponent = -kDpExponentBias; - static const uint64_t kDpExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); - static const uint64_t kDpSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); - static const uint64_t kDpHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); - - uint64_t f; - int e; -}; - -inline DiyFp GetCachedPower(int e, int* K) { - // 10^-348, 10^-340, ..., 10^340 - static const uint64_t kCachedPowers_F[] = { - RAPIDJSON_UINT64_C2(0xfa8fd5a0, 0x081c0288), RAPIDJSON_UINT64_C2(0xbaaee17f, 0xa23ebf76), - RAPIDJSON_UINT64_C2(0x8b16fb20, 0x3055ac76), RAPIDJSON_UINT64_C2(0xcf42894a, 0x5dce35ea), - RAPIDJSON_UINT64_C2(0x9a6bb0aa, 0x55653b2d), RAPIDJSON_UINT64_C2(0xe61acf03, 0x3d1a45df), - RAPIDJSON_UINT64_C2(0xab70fe17, 0xc79ac6ca), RAPIDJSON_UINT64_C2(0xff77b1fc, 0xbebcdc4f), - RAPIDJSON_UINT64_C2(0xbe5691ef, 0x416bd60c), RAPIDJSON_UINT64_C2(0x8dd01fad, 0x907ffc3c), - RAPIDJSON_UINT64_C2(0xd3515c28, 0x31559a83), RAPIDJSON_UINT64_C2(0x9d71ac8f, 0xada6c9b5), - RAPIDJSON_UINT64_C2(0xea9c2277, 0x23ee8bcb), RAPIDJSON_UINT64_C2(0xaecc4991, 0x4078536d), - RAPIDJSON_UINT64_C2(0x823c1279, 0x5db6ce57), RAPIDJSON_UINT64_C2(0xc2109436, 0x4dfb5637), - RAPIDJSON_UINT64_C2(0x9096ea6f, 0x3848984f), RAPIDJSON_UINT64_C2(0xd77485cb, 0x25823ac7), - RAPIDJSON_UINT64_C2(0xa086cfcd, 0x97bf97f4), RAPIDJSON_UINT64_C2(0xef340a98, 0x172aace5), - RAPIDJSON_UINT64_C2(0xb23867fb, 0x2a35b28e), RAPIDJSON_UINT64_C2(0x84c8d4df, 0xd2c63f3b), - RAPIDJSON_UINT64_C2(0xc5dd4427, 0x1ad3cdba), RAPIDJSON_UINT64_C2(0x936b9fce, 0xbb25c996), - RAPIDJSON_UINT64_C2(0xdbac6c24, 0x7d62a584), RAPIDJSON_UINT64_C2(0xa3ab6658, 0x0d5fdaf6), - RAPIDJSON_UINT64_C2(0xf3e2f893, 0xdec3f126), RAPIDJSON_UINT64_C2(0xb5b5ada8, 0xaaff80b8), - RAPIDJSON_UINT64_C2(0x87625f05, 0x6c7c4a8b), RAPIDJSON_UINT64_C2(0xc9bcff60, 0x34c13053), - RAPIDJSON_UINT64_C2(0x964e858c, 0x91ba2655), RAPIDJSON_UINT64_C2(0xdff97724, 0x70297ebd), - RAPIDJSON_UINT64_C2(0xa6dfbd9f, 0xb8e5b88f), RAPIDJSON_UINT64_C2(0xf8a95fcf, 0x88747d94), - RAPIDJSON_UINT64_C2(0xb9447093, 0x8fa89bcf), RAPIDJSON_UINT64_C2(0x8a08f0f8, 0xbf0f156b), - RAPIDJSON_UINT64_C2(0xcdb02555, 0x653131b6), RAPIDJSON_UINT64_C2(0x993fe2c6, 0xd07b7fac), - RAPIDJSON_UINT64_C2(0xe45c10c4, 0x2a2b3b06), RAPIDJSON_UINT64_C2(0xaa242499, 0x697392d3), - RAPIDJSON_UINT64_C2(0xfd87b5f2, 0x8300ca0e), RAPIDJSON_UINT64_C2(0xbce50864, 0x92111aeb), - RAPIDJSON_UINT64_C2(0x8cbccc09, 0x6f5088cc), RAPIDJSON_UINT64_C2(0xd1b71758, 0xe219652c), - RAPIDJSON_UINT64_C2(0x9c400000, 0x00000000), RAPIDJSON_UINT64_C2(0xe8d4a510, 0x00000000), - RAPIDJSON_UINT64_C2(0xad78ebc5, 0xac620000), RAPIDJSON_UINT64_C2(0x813f3978, 0xf8940984), - RAPIDJSON_UINT64_C2(0xc097ce7b, 0xc90715b3), RAPIDJSON_UINT64_C2(0x8f7e32ce, 0x7bea5c70), - RAPIDJSON_UINT64_C2(0xd5d238a4, 0xabe98068), RAPIDJSON_UINT64_C2(0x9f4f2726, 0x179a2245), - RAPIDJSON_UINT64_C2(0xed63a231, 0xd4c4fb27), RAPIDJSON_UINT64_C2(0xb0de6538, 0x8cc8ada8), - RAPIDJSON_UINT64_C2(0x83c7088e, 0x1aab65db), RAPIDJSON_UINT64_C2(0xc45d1df9, 0x42711d9a), - RAPIDJSON_UINT64_C2(0x924d692c, 0xa61be758), RAPIDJSON_UINT64_C2(0xda01ee64, 0x1a708dea), - RAPIDJSON_UINT64_C2(0xa26da399, 0x9aef774a), RAPIDJSON_UINT64_C2(0xf209787b, 0xb47d6b85), - RAPIDJSON_UINT64_C2(0xb454e4a1, 0x79dd1877), RAPIDJSON_UINT64_C2(0x865b8692, 0x5b9bc5c2), - RAPIDJSON_UINT64_C2(0xc83553c5, 0xc8965d3d), RAPIDJSON_UINT64_C2(0x952ab45c, 0xfa97a0b3), - RAPIDJSON_UINT64_C2(0xde469fbd, 0x99a05fe3), RAPIDJSON_UINT64_C2(0xa59bc234, 0xdb398c25), - RAPIDJSON_UINT64_C2(0xf6c69a72, 0xa3989f5c), RAPIDJSON_UINT64_C2(0xb7dcbf53, 0x54e9bece), - RAPIDJSON_UINT64_C2(0x88fcf317, 0xf22241e2), RAPIDJSON_UINT64_C2(0xcc20ce9b, 0xd35c78a5), - RAPIDJSON_UINT64_C2(0x98165af3, 0x7b2153df), RAPIDJSON_UINT64_C2(0xe2a0b5dc, 0x971f303a), - RAPIDJSON_UINT64_C2(0xa8d9d153, 0x5ce3b396), RAPIDJSON_UINT64_C2(0xfb9b7cd9, 0xa4a7443c), - RAPIDJSON_UINT64_C2(0xbb764c4c, 0xa7a44410), RAPIDJSON_UINT64_C2(0x8bab8eef, 0xb6409c1a), - RAPIDJSON_UINT64_C2(0xd01fef10, 0xa657842c), RAPIDJSON_UINT64_C2(0x9b10a4e5, 0xe9913129), - RAPIDJSON_UINT64_C2(0xe7109bfb, 0xa19c0c9d), RAPIDJSON_UINT64_C2(0xac2820d9, 0x623bf429), - RAPIDJSON_UINT64_C2(0x80444b5e, 0x7aa7cf85), RAPIDJSON_UINT64_C2(0xbf21e440, 0x03acdd2d), - RAPIDJSON_UINT64_C2(0x8e679c2f, 0x5e44ff8f), RAPIDJSON_UINT64_C2(0xd433179d, 0x9c8cb841), - RAPIDJSON_UINT64_C2(0x9e19db92, 0xb4e31ba9), RAPIDJSON_UINT64_C2(0xeb96bf6e, 0xbadf77d9), - RAPIDJSON_UINT64_C2(0xaf87023b, 0x9bf0ee6b) - }; - static const int16_t kCachedPowers_E[] = { - -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, - -954, -927, -901, -874, -847, -821, -794, -768, -741, -715, - -688, -661, -635, -608, -582, -555, -529, -502, -475, -449, - -422, -396, -369, -343, -316, -289, -263, -236, -210, -183, - -157, -130, -103, -77, -50, -24, 3, 30, 56, 83, - 109, 136, 162, 189, 216, 242, 269, 295, 322, 348, - 375, 402, 428, 455, 481, 508, 534, 561, 588, 614, - 641, 667, 694, 720, 747, 774, 800, 827, 853, 880, - 907, 933, 960, 986, 1013, 1039, 1066 - }; - - //int k = static_cast(ceil((-61 - e) * 0.30102999566398114)) + 374; - double dk = (-61 - e) * 0.30102999566398114 + 347; // dk must be positive, so can do ceiling in positive - int k = static_cast(dk); - if (k != dk) - k++; - - unsigned index = static_cast((k >> 3) + 1); - *K = -(-348 + static_cast(index << 3)); // decimal exponent no need lookup table - - return DiyFp(kCachedPowers_F[index], kCachedPowers_E[index]); -} - -inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t wp_w) { - while (rest < wp_w && delta - rest >= ten_kappa && - (rest + ten_kappa < wp_w || /// closer - wp_w - rest > rest + ten_kappa - wp_w)) { - buffer[len - 1]--; - rest += ten_kappa; - } -} - -inline unsigned CountDecimalDigit32(uint32_t n) { - // Simple pure C++ implementation was faster than __builtin_clz version in this situation. - if (n < 10) return 1; - if (n < 100) return 2; - if (n < 1000) return 3; - if (n < 10000) return 4; - if (n < 100000) return 5; - if (n < 1000000) return 6; - if (n < 10000000) return 7; - if (n < 100000000) return 8; - if (n < 1000000000) return 9; - return 10; -} - -inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buffer, int* len, int* K) { - static const uint32_t kPow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; - const DiyFp one(uint64_t(1) << -Mp.e, Mp.e); - const DiyFp wp_w = Mp - W; - uint32_t p1 = static_cast(Mp.f >> -one.e); - uint64_t p2 = Mp.f & (one.f - 1); - int kappa = CountDecimalDigit32(p1); - *len = 0; - - while (kappa > 0) { - uint32_t d; - switch (kappa) { - case 10: d = p1 / 1000000000; p1 %= 1000000000; break; - case 9: d = p1 / 100000000; p1 %= 100000000; break; - case 8: d = p1 / 10000000; p1 %= 10000000; break; - case 7: d = p1 / 1000000; p1 %= 1000000; break; - case 6: d = p1 / 100000; p1 %= 100000; break; - case 5: d = p1 / 10000; p1 %= 10000; break; - case 4: d = p1 / 1000; p1 %= 1000; break; - case 3: d = p1 / 100; p1 %= 100; break; - case 2: d = p1 / 10; p1 %= 10; break; - case 1: d = p1; p1 = 0; break; - default: -#if defined(_MSC_VER) - __assume(0); -#elif __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5) - __builtin_unreachable(); -#else - d = 0; -#endif - } - if (d || *len) - buffer[(*len)++] = static_cast('0' + static_cast(d)); - kappa--; - uint64_t tmp = (static_cast(p1) << -one.e) + p2; - if (tmp <= delta) { - *K += kappa; - GrisuRound(buffer, *len, delta, tmp, static_cast(kPow10[kappa]) << -one.e, wp_w.f); - return; - } - } - - // kappa = 0 - for (;;) { - p2 *= 10; - delta *= 10; - char d = static_cast(p2 >> -one.e); - if (d || *len) - buffer[(*len)++] = static_cast('0' + d); - p2 &= one.f - 1; - kappa--; - if (p2 < delta) { - *K += kappa; - GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * kPow10[-kappa]); - return; - } - } -} - -inline void Grisu2(double value, char* buffer, int* length, int* K) { - const DiyFp v(value); - DiyFp w_m, w_p; - v.NormalizedBoundaries(&w_m, &w_p); - - const DiyFp c_mk = GetCachedPower(w_p.e, K); - const DiyFp W = v.Normalize() * c_mk; - DiyFp Wp = w_p * c_mk; - DiyFp Wm = w_m * c_mk; - Wm.f++; - Wp.f--; - DigitGen(W, Wp, Wp.f - Wm.f, buffer, length, K); -} - -inline char* WriteExponent(int K, char* buffer) { - if (K < 0) { - *buffer++ = '-'; - K = -K; - } - - if (K >= 100) { - *buffer++ = static_cast('0' + static_cast(K / 100)); - K %= 100; - const char* d = GetDigitsLut() + K * 2; - *buffer++ = d[0]; - *buffer++ = d[1]; - } - else if (K >= 10) { - const char* d = GetDigitsLut() + K * 2; - *buffer++ = d[0]; - *buffer++ = d[1]; - } - else - *buffer++ = static_cast('0' + static_cast(K)); - - return buffer; -} - -inline char* Prettify(char* buffer, int length, int k) { - const int kk = length + k; // 10^(kk-1) <= v < 10^kk - - if (length <= kk && kk <= 21) { - // 1234e7 -> 12340000000 - for (int i = length; i < kk; i++) - buffer[i] = '0'; - buffer[kk] = '.'; - buffer[kk + 1] = '0'; - return &buffer[kk + 2]; - } - else if (0 < kk && kk <= 21) { - // 1234e-2 -> 12.34 - std::memmove(&buffer[kk + 1], &buffer[kk], length - kk); - buffer[kk] = '.'; - return &buffer[length + 1]; - } - else if (-6 < kk && kk <= 0) { - // 1234e-6 -> 0.001234 - const int offset = 2 - kk; - std::memmove(&buffer[offset], &buffer[0], length); - buffer[0] = '0'; - buffer[1] = '.'; - for (int i = 2; i < offset; i++) - buffer[i] = '0'; - return &buffer[length + offset]; - } - else if (length == 1) { - // 1e30 - buffer[1] = 'e'; - return WriteExponent(kk - 1, &buffer[2]); - } - else { - // 1234e30 -> 1.234e33 - std::memmove(&buffer[2], &buffer[1], length - 1); - buffer[1] = '.'; - buffer[length + 1] = 'e'; - return WriteExponent(kk - 1, &buffer[0 + length + 2]); - } -} - -inline char* dtoa(double value, char* buffer) { - if (value == 0) { - buffer[0] = '0'; - buffer[1] = '.'; - buffer[2] = '0'; - return &buffer[3]; - } - else { - if (value < 0) { - *buffer++ = '-'; - value = -value; - } - int length, K; - Grisu2(value, buffer, &length, &K); - return Prettify(buffer, length, K); - } -} - -#ifdef __GNUC__ -RAPIDJSON_DIAG_POP -#endif - -} // namespace internal -} // namespace rapidjson - -#endif // RAPIDJSON_DTOA_ diff --git a/tools/optimizer/rapidjson/internal/itoa.h b/tools/optimizer/rapidjson/internal/itoa.h deleted file mode 100644 index 425e9830c04a6..0000000000000 --- a/tools/optimizer/rapidjson/internal/itoa.h +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_ITOA_ -#define RAPIDJSON_ITOA_ - -namespace rapidjson { -namespace internal { - -inline const char* GetDigitsLut() { - static const char cDigitsLut[200] = { - '0','0','0','1','0','2','0','3','0','4','0','5','0','6','0','7','0','8','0','9', - '1','0','1','1','1','2','1','3','1','4','1','5','1','6','1','7','1','8','1','9', - '2','0','2','1','2','2','2','3','2','4','2','5','2','6','2','7','2','8','2','9', - '3','0','3','1','3','2','3','3','3','4','3','5','3','6','3','7','3','8','3','9', - '4','0','4','1','4','2','4','3','4','4','4','5','4','6','4','7','4','8','4','9', - '5','0','5','1','5','2','5','3','5','4','5','5','5','6','5','7','5','8','5','9', - '6','0','6','1','6','2','6','3','6','4','6','5','6','6','6','7','6','8','6','9', - '7','0','7','1','7','2','7','3','7','4','7','5','7','6','7','7','7','8','7','9', - '8','0','8','1','8','2','8','3','8','4','8','5','8','6','8','7','8','8','8','9', - '9','0','9','1','9','2','9','3','9','4','9','5','9','6','9','7','9','8','9','9' - }; - return cDigitsLut; -} - -inline char* u32toa(uint32_t value, char* buffer) { - const char* cDigitsLut = GetDigitsLut(); - - if (value < 10000) { - const uint32_t d1 = (value / 100) << 1; - const uint32_t d2 = (value % 100) << 1; - - if (value >= 1000) - *buffer++ = cDigitsLut[d1]; - if (value >= 100) - *buffer++ = cDigitsLut[d1 + 1]; - if (value >= 10) - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - } - else if (value < 100000000) { - // value = bbbbcccc - const uint32_t b = value / 10000; - const uint32_t c = value % 10000; - - const uint32_t d1 = (b / 100) << 1; - const uint32_t d2 = (b % 100) << 1; - - const uint32_t d3 = (c / 100) << 1; - const uint32_t d4 = (c % 100) << 1; - - if (value >= 10000000) - *buffer++ = cDigitsLut[d1]; - if (value >= 1000000) - *buffer++ = cDigitsLut[d1 + 1]; - if (value >= 100000) - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - - *buffer++ = cDigitsLut[d3]; - *buffer++ = cDigitsLut[d3 + 1]; - *buffer++ = cDigitsLut[d4]; - *buffer++ = cDigitsLut[d4 + 1]; - } - else { - // value = aabbbbcccc in decimal - - const uint32_t a = value / 100000000; // 1 to 42 - value %= 100000000; - - if (a >= 10) { - const unsigned i = a << 1; - *buffer++ = cDigitsLut[i]; - *buffer++ = cDigitsLut[i + 1]; - } - else - *buffer++ = static_cast('0' + static_cast(a)); - - const uint32_t b = value / 10000; // 0 to 9999 - const uint32_t c = value % 10000; // 0 to 9999 - - const uint32_t d1 = (b / 100) << 1; - const uint32_t d2 = (b % 100) << 1; - - const uint32_t d3 = (c / 100) << 1; - const uint32_t d4 = (c % 100) << 1; - - *buffer++ = cDigitsLut[d1]; - *buffer++ = cDigitsLut[d1 + 1]; - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - *buffer++ = cDigitsLut[d3]; - *buffer++ = cDigitsLut[d3 + 1]; - *buffer++ = cDigitsLut[d4]; - *buffer++ = cDigitsLut[d4 + 1]; - } - return buffer; -} - -inline char* i32toa(int32_t value, char* buffer) { - if (value < 0) { - *buffer++ = '-'; - value = -value; - } - - return u32toa(static_cast(value), buffer); -} - -inline char* u64toa(uint64_t value, char* buffer) { - const char* cDigitsLut = GetDigitsLut(); - const uint64_t kTen8 = 100000000; - const uint64_t kTen9 = kTen8 * 10; - const uint64_t kTen10 = kTen8 * 100; - const uint64_t kTen11 = kTen8 * 1000; - const uint64_t kTen12 = kTen8 * 10000; - const uint64_t kTen13 = kTen8 * 100000; - const uint64_t kTen14 = kTen8 * 1000000; - const uint64_t kTen15 = kTen8 * 10000000; - const uint64_t kTen16 = kTen8 * kTen8; - - if (value < kTen8) { - uint32_t v = static_cast(value); - if (v < 10000) { - const uint32_t d1 = (v / 100) << 1; - const uint32_t d2 = (v % 100) << 1; - - if (v >= 1000) - *buffer++ = cDigitsLut[d1]; - if (v >= 100) - *buffer++ = cDigitsLut[d1 + 1]; - if (v >= 10) - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - } - else { - // value = bbbbcccc - const uint32_t b = v / 10000; - const uint32_t c = v % 10000; - - const uint32_t d1 = (b / 100) << 1; - const uint32_t d2 = (b % 100) << 1; - - const uint32_t d3 = (c / 100) << 1; - const uint32_t d4 = (c % 100) << 1; - - if (value >= 10000000) - *buffer++ = cDigitsLut[d1]; - if (value >= 1000000) - *buffer++ = cDigitsLut[d1 + 1]; - if (value >= 100000) - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - - *buffer++ = cDigitsLut[d3]; - *buffer++ = cDigitsLut[d3 + 1]; - *buffer++ = cDigitsLut[d4]; - *buffer++ = cDigitsLut[d4 + 1]; - } - } - else if (value < kTen16) { - const uint32_t v0 = static_cast(value / kTen8); - const uint32_t v1 = static_cast(value % kTen8); - - const uint32_t b0 = v0 / 10000; - const uint32_t c0 = v0 % 10000; - - const uint32_t d1 = (b0 / 100) << 1; - const uint32_t d2 = (b0 % 100) << 1; - - const uint32_t d3 = (c0 / 100) << 1; - const uint32_t d4 = (c0 % 100) << 1; - - const uint32_t b1 = v1 / 10000; - const uint32_t c1 = v1 % 10000; - - const uint32_t d5 = (b1 / 100) << 1; - const uint32_t d6 = (b1 % 100) << 1; - - const uint32_t d7 = (c1 / 100) << 1; - const uint32_t d8 = (c1 % 100) << 1; - - if (value >= kTen15) - *buffer++ = cDigitsLut[d1]; - if (value >= kTen14) - *buffer++ = cDigitsLut[d1 + 1]; - if (value >= kTen13) - *buffer++ = cDigitsLut[d2]; - if (value >= kTen12) - *buffer++ = cDigitsLut[d2 + 1]; - if (value >= kTen11) - *buffer++ = cDigitsLut[d3]; - if (value >= kTen10) - *buffer++ = cDigitsLut[d3 + 1]; - if (value >= kTen9) - *buffer++ = cDigitsLut[d4]; - if (value >= kTen8) - *buffer++ = cDigitsLut[d4 + 1]; - - *buffer++ = cDigitsLut[d5]; - *buffer++ = cDigitsLut[d5 + 1]; - *buffer++ = cDigitsLut[d6]; - *buffer++ = cDigitsLut[d6 + 1]; - *buffer++ = cDigitsLut[d7]; - *buffer++ = cDigitsLut[d7 + 1]; - *buffer++ = cDigitsLut[d8]; - *buffer++ = cDigitsLut[d8 + 1]; - } - else { - const uint32_t a = static_cast(value / kTen16); // 1 to 1844 - value %= kTen16; - - if (a < 10) - *buffer++ = static_cast('0' + static_cast(a)); - else if (a < 100) { - const uint32_t i = a << 1; - *buffer++ = cDigitsLut[i]; - *buffer++ = cDigitsLut[i + 1]; - } - else if (a < 1000) { - *buffer++ = static_cast('0' + static_cast(a / 100)); - - const uint32_t i = (a % 100) << 1; - *buffer++ = cDigitsLut[i]; - *buffer++ = cDigitsLut[i + 1]; - } - else { - const uint32_t i = (a / 100) << 1; - const uint32_t j = (a % 100) << 1; - *buffer++ = cDigitsLut[i]; - *buffer++ = cDigitsLut[i + 1]; - *buffer++ = cDigitsLut[j]; - *buffer++ = cDigitsLut[j + 1]; - } - - const uint32_t v0 = static_cast(value / kTen8); - const uint32_t v1 = static_cast(value % kTen8); - - const uint32_t b0 = v0 / 10000; - const uint32_t c0 = v0 % 10000; - - const uint32_t d1 = (b0 / 100) << 1; - const uint32_t d2 = (b0 % 100) << 1; - - const uint32_t d3 = (c0 / 100) << 1; - const uint32_t d4 = (c0 % 100) << 1; - - const uint32_t b1 = v1 / 10000; - const uint32_t c1 = v1 % 10000; - - const uint32_t d5 = (b1 / 100) << 1; - const uint32_t d6 = (b1 % 100) << 1; - - const uint32_t d7 = (c1 / 100) << 1; - const uint32_t d8 = (c1 % 100) << 1; - - *buffer++ = cDigitsLut[d1]; - *buffer++ = cDigitsLut[d1 + 1]; - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - *buffer++ = cDigitsLut[d3]; - *buffer++ = cDigitsLut[d3 + 1]; - *buffer++ = cDigitsLut[d4]; - *buffer++ = cDigitsLut[d4 + 1]; - *buffer++ = cDigitsLut[d5]; - *buffer++ = cDigitsLut[d5 + 1]; - *buffer++ = cDigitsLut[d6]; - *buffer++ = cDigitsLut[d6 + 1]; - *buffer++ = cDigitsLut[d7]; - *buffer++ = cDigitsLut[d7 + 1]; - *buffer++ = cDigitsLut[d8]; - *buffer++ = cDigitsLut[d8 + 1]; - } - - return buffer; -} - -inline char* i64toa(int64_t value, char* buffer) { - if (value < 0) { - *buffer++ = '-'; - value = -value; - } - - return u64toa(static_cast(value), buffer); -} - -} // namespace internal -} // namespace rapidjson - -#endif // RAPIDJSON_ITOA_ diff --git a/tools/optimizer/rapidjson/internal/meta.h b/tools/optimizer/rapidjson/internal/meta.h deleted file mode 100644 index dbe5450d6b9be..0000000000000 --- a/tools/optimizer/rapidjson/internal/meta.h +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_INTERNAL_META_H_ -#define RAPIDJSON_INTERNAL_META_H_ - -#ifndef RAPIDJSON_RAPIDJSON_H_ -#error not yet included. Do not include this file directly. -#endif - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#endif -#if defined(_MSC_VER) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(6334) -#endif - -#if RAPIDJSON_HAS_CXX11_TYPETRAITS -#include -#endif - -//@cond RAPIDJSON_INTERNAL -namespace rapidjson { -namespace internal { - -// Helper to wrap/convert arbitrary types to void, useful for arbitrary type matching -template struct Void { typedef void Type; }; - -/////////////////////////////////////////////////////////////////////////////// -// BoolType, TrueType, FalseType -// -template struct BoolType { - static const bool Value = Cond; - typedef BoolType Type; -}; -typedef BoolType TrueType; -typedef BoolType FalseType; - - -/////////////////////////////////////////////////////////////////////////////// -// SelectIf, BoolExpr, NotExpr, AndExpr, OrExpr -// - -template struct SelectIfImpl { template struct Apply { typedef T1 Type; }; }; -template <> struct SelectIfImpl { template struct Apply { typedef T2 Type; }; }; -template struct SelectIfCond : SelectIfImpl::template Apply {}; -template struct SelectIf : SelectIfCond {}; - -template struct AndExprCond : FalseType {}; -template <> struct AndExprCond : TrueType {}; -template struct OrExprCond : TrueType {}; -template <> struct OrExprCond : FalseType {}; - -template struct BoolExpr : SelectIf::Type {}; -template struct NotExpr : SelectIf::Type {}; -template struct AndExpr : AndExprCond::Type {}; -template struct OrExpr : OrExprCond::Type {}; - - -/////////////////////////////////////////////////////////////////////////////// -// AddConst, MaybeAddConst, RemoveConst -template struct AddConst { typedef const T Type; }; -template struct MaybeAddConst : SelectIfCond {}; -template struct RemoveConst { typedef T Type; }; -template struct RemoveConst { typedef T Type; }; - - -/////////////////////////////////////////////////////////////////////////////// -// IsSame, IsConst, IsMoreConst, IsPointer -// -template struct IsSame : FalseType {}; -template struct IsSame : TrueType {}; - -template struct IsConst : FalseType {}; -template struct IsConst : TrueType {}; - -template -struct IsMoreConst - : AndExpr::Type, typename RemoveConst::Type>, - BoolType::Value >= IsConst::Value> >::Type {}; - -template struct IsPointer : FalseType {}; -template struct IsPointer : TrueType {}; - -/////////////////////////////////////////////////////////////////////////////// -// IsBaseOf -// -#if RAPIDJSON_HAS_CXX11_TYPETRAITS - -template struct IsBaseOf - : BoolType< ::std::is_base_of::value> {}; - -#else // simplified version adopted from Boost - -template struct IsBaseOfImpl { - RAPIDJSON_STATIC_ASSERT(sizeof(B) != 0); - RAPIDJSON_STATIC_ASSERT(sizeof(D) != 0); - - typedef char (&Yes)[1]; - typedef char (&No) [2]; - - template - static Yes Check(const D*, T); - static No Check(const B*, int); - - struct Host { - operator const B*() const; - operator const D*(); - }; - - enum { Value = (sizeof(Check(Host(), 0)) == sizeof(Yes)) }; -}; - -template struct IsBaseOf - : OrExpr, BoolExpr > >::Type {}; - -#endif // RAPIDJSON_HAS_CXX11_TYPETRAITS - - -////////////////////////////////////////////////////////////////////////// -// EnableIf / DisableIf -// -template struct EnableIfCond { typedef T Type; }; -template struct EnableIfCond { /* empty */ }; - -template struct DisableIfCond { typedef T Type; }; -template struct DisableIfCond { /* empty */ }; - -template -struct EnableIf : EnableIfCond {}; - -template -struct DisableIf : DisableIfCond {}; - -// SFINAE helpers -struct SfinaeTag {}; -template struct RemoveSfinaeTag; -template struct RemoveSfinaeTag { typedef T Type; }; - -#define RAPIDJSON_REMOVEFPTR_(type) \ - typename ::rapidjson::internal::RemoveSfinaeTag \ - < ::rapidjson::internal::SfinaeTag&(*) type>::Type - -#define RAPIDJSON_ENABLEIF(cond) \ - typename ::rapidjson::internal::EnableIf \ - ::Type * = NULL - -#define RAPIDJSON_DISABLEIF(cond) \ - typename ::rapidjson::internal::DisableIf \ - ::Type * = NULL - -#define RAPIDJSON_ENABLEIF_RETURN(cond,returntype) \ - typename ::rapidjson::internal::EnableIf \ - ::Type - -#define RAPIDJSON_DISABLEIF_RETURN(cond,returntype) \ - typename ::rapidjson::internal::DisableIf \ - ::Type - -} // namespace internal -} // namespace rapidjson -//@endcond - -#if defined(__GNUC__) || defined(_MSC_VER) -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_INTERNAL_META_H_ diff --git a/tools/optimizer/rapidjson/internal/pow10.h b/tools/optimizer/rapidjson/internal/pow10.h deleted file mode 100644 index 72e0dac9f6bb2..0000000000000 --- a/tools/optimizer/rapidjson/internal/pow10.h +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_POW10_ -#define RAPIDJSON_POW10_ - -namespace rapidjson { -namespace internal { - -//! Computes integer powers of 10 in double (10.0^n). -/*! This function uses lookup table for fast and accurate results. - \param n non-negative exponent. Must <= 308. - \return 10.0^n -*/ -inline double Pow10(int n) { - static const double e[] = { // 1e-0...1e308: 309 * 8 bytes = 2472 bytes - 1e+0, - 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, - 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, - 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, - 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, - 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, - 1e+101,1e+102,1e+103,1e+104,1e+105,1e+106,1e+107,1e+108,1e+109,1e+110,1e+111,1e+112,1e+113,1e+114,1e+115,1e+116,1e+117,1e+118,1e+119,1e+120, - 1e+121,1e+122,1e+123,1e+124,1e+125,1e+126,1e+127,1e+128,1e+129,1e+130,1e+131,1e+132,1e+133,1e+134,1e+135,1e+136,1e+137,1e+138,1e+139,1e+140, - 1e+141,1e+142,1e+143,1e+144,1e+145,1e+146,1e+147,1e+148,1e+149,1e+150,1e+151,1e+152,1e+153,1e+154,1e+155,1e+156,1e+157,1e+158,1e+159,1e+160, - 1e+161,1e+162,1e+163,1e+164,1e+165,1e+166,1e+167,1e+168,1e+169,1e+170,1e+171,1e+172,1e+173,1e+174,1e+175,1e+176,1e+177,1e+178,1e+179,1e+180, - 1e+181,1e+182,1e+183,1e+184,1e+185,1e+186,1e+187,1e+188,1e+189,1e+190,1e+191,1e+192,1e+193,1e+194,1e+195,1e+196,1e+197,1e+198,1e+199,1e+200, - 1e+201,1e+202,1e+203,1e+204,1e+205,1e+206,1e+207,1e+208,1e+209,1e+210,1e+211,1e+212,1e+213,1e+214,1e+215,1e+216,1e+217,1e+218,1e+219,1e+220, - 1e+221,1e+222,1e+223,1e+224,1e+225,1e+226,1e+227,1e+228,1e+229,1e+230,1e+231,1e+232,1e+233,1e+234,1e+235,1e+236,1e+237,1e+238,1e+239,1e+240, - 1e+241,1e+242,1e+243,1e+244,1e+245,1e+246,1e+247,1e+248,1e+249,1e+250,1e+251,1e+252,1e+253,1e+254,1e+255,1e+256,1e+257,1e+258,1e+259,1e+260, - 1e+261,1e+262,1e+263,1e+264,1e+265,1e+266,1e+267,1e+268,1e+269,1e+270,1e+271,1e+272,1e+273,1e+274,1e+275,1e+276,1e+277,1e+278,1e+279,1e+280, - 1e+281,1e+282,1e+283,1e+284,1e+285,1e+286,1e+287,1e+288,1e+289,1e+290,1e+291,1e+292,1e+293,1e+294,1e+295,1e+296,1e+297,1e+298,1e+299,1e+300, - 1e+301,1e+302,1e+303,1e+304,1e+305,1e+306,1e+307,1e+308 - }; - RAPIDJSON_ASSERT(n >= 0 && n <= 308); - return e[n]; -} - -} // namespace internal -} // namespace rapidjson - -#endif // RAPIDJSON_POW10_ diff --git a/tools/optimizer/rapidjson/internal/stack.h b/tools/optimizer/rapidjson/internal/stack.h deleted file mode 100644 index ea722c6f1918a..0000000000000 --- a/tools/optimizer/rapidjson/internal/stack.h +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_INTERNAL_STACK_H_ -#define RAPIDJSON_INTERNAL_STACK_H_ - -namespace rapidjson { -namespace internal { - -/////////////////////////////////////////////////////////////////////////////// -// Stack - -//! A type-unsafe stack for storing different types of data. -/*! \tparam Allocator Allocator for allocating stack memory. -*/ -template -class Stack { -public: - // Optimization note: Do not allocate memory for stack_ in constructor. - // Do it lazily when first Push() -> Expand() -> Resize(). - Stack(Allocator* allocator, size_t stackCapacity) : allocator_(allocator), ownAllocator(0), stack_(0), stackTop_(0), stackEnd_(0), initialCapacity_(stackCapacity) { - RAPIDJSON_ASSERT(stackCapacity > 0); - if (!allocator_) - ownAllocator = allocator_ = new Allocator(); - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - Stack(Stack&& rhs) - : allocator_(rhs.allocator_), - ownAllocator(rhs.ownAllocator), - stack_(rhs.stack_), - stackTop_(rhs.stackTop_), - stackEnd_(rhs.stackEnd_), - initialCapacity_(rhs.initialCapacity_) - { - rhs.allocator_ = 0; - rhs.ownAllocator = 0; - rhs.stack_ = 0; - rhs.stackTop_ = 0; - rhs.stackEnd_ = 0; - rhs.initialCapacity_ = 0; - } -#endif - - ~Stack() { - Destroy(); - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - Stack& operator=(Stack&& rhs) { - if (&rhs != this) - { - Destroy(); - - allocator_ = rhs.allocator_; - ownAllocator = rhs.ownAllocator; - stack_ = rhs.stack_; - stackTop_ = rhs.stackTop_; - stackEnd_ = rhs.stackEnd_; - initialCapacity_ = rhs.initialCapacity_; - - rhs.allocator_ = 0; - rhs.ownAllocator = 0; - rhs.stack_ = 0; - rhs.stackTop_ = 0; - rhs.stackEnd_ = 0; - rhs.initialCapacity_ = 0; - } - return *this; - } -#endif - - void Clear() { stackTop_ = stack_; } - - void ShrinkToFit() { - if (Empty()) { - // If the stack is empty, completely deallocate the memory. - Allocator::Free(stack_); - stack_ = 0; - stackTop_ = 0; - stackEnd_ = 0; - } - else - Resize(GetSize()); - } - - // Optimization note: try to minimize the size of this function for force inline. - // Expansion is run very infrequently, so it is moved to another (probably non-inline) function. - template - RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) { - // Expand the stack if needed - if (stackTop_ + sizeof(T) * count >= stackEnd_) - Expand(count); - - T* ret = reinterpret_cast(stackTop_); - stackTop_ += sizeof(T) * count; - return ret; - } - - template - T* Pop(size_t count) { - RAPIDJSON_ASSERT(GetSize() >= count * sizeof(T)); - stackTop_ -= count * sizeof(T); - return reinterpret_cast(stackTop_); - } - - template - T* Top() { - RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); - return reinterpret_cast(stackTop_ - sizeof(T)); - } - - template - T* Bottom() { return (T*)stack_; } - - Allocator& GetAllocator() { return *allocator_; } - bool Empty() const { return stackTop_ == stack_; } - size_t GetSize() const { return static_cast(stackTop_ - stack_); } - size_t GetCapacity() const { return static_cast(stackEnd_ - stack_); } - -private: - template - void Expand(size_t count) { - // Only expand the capacity if the current stack exists. Otherwise just create a stack with initial capacity. - size_t newCapacity; - if (stack_ == 0) - newCapacity = initialCapacity_; - else { - newCapacity = GetCapacity(); - newCapacity += (newCapacity + 1) / 2; - } - size_t newSize = GetSize() + sizeof(T) * count; - if (newCapacity < newSize) - newCapacity = newSize; - - Resize(newCapacity); - } - - void Resize(size_t newCapacity) { - const size_t size = GetSize(); // Backup the current size - stack_ = (char*)allocator_->Realloc(stack_, GetCapacity(), newCapacity); - stackTop_ = stack_ + size; - stackEnd_ = stack_ + newCapacity; - } - - void Destroy() { - Allocator::Free(stack_); - delete ownAllocator; // Only delete if it is owned by the stack - } - - // Prohibit copy constructor & assignment operator. - Stack(const Stack&); - Stack& operator=(const Stack&); - - Allocator* allocator_; - Allocator* ownAllocator; - char *stack_; - char *stackTop_; - char *stackEnd_; - size_t initialCapacity_; -}; - -} // namespace internal -} // namespace rapidjson - -#endif // RAPIDJSON_STACK_H_ diff --git a/tools/optimizer/rapidjson/internal/strfunc.h b/tools/optimizer/rapidjson/internal/strfunc.h deleted file mode 100644 index 80adcb6b730a1..0000000000000 --- a/tools/optimizer/rapidjson/internal/strfunc.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ -#define RAPIDJSON_INTERNAL_STRFUNC_H_ - -namespace rapidjson { -namespace internal { - -//! Custom strlen() which works on different character types. -/*! \tparam Ch Character type (e.g. char, wchar_t, short) - \param s Null-terminated input string. - \return Number of characters in the string. - \note This has the same semantics as strlen(), the return value is not number of Unicode codepoints. -*/ -template -inline SizeType StrLen(const Ch* s) { - const Ch* p = s; - while (*p) ++p; - return SizeType(p - s); -} - -} // namespace internal -} // namespace rapidjson - -#endif // RAPIDJSON_INTERNAL_STRFUNC_H_ diff --git a/tools/optimizer/rapidjson/memorybuffer.h b/tools/optimizer/rapidjson/memorybuffer.h deleted file mode 100644 index ef15c46780e19..0000000000000 --- a/tools/optimizer/rapidjson/memorybuffer.h +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_MEMORYBUFFER_H_ -#define RAPIDJSON_MEMORYBUFFER_H_ - -#include "rapidjson.h" -#include "internal/stack.h" - -namespace rapidjson { - -//! Represents an in-memory output byte stream. -/*! - This class is mainly for being wrapped by EncodedOutputStream or AutoUTFOutputStream. - - It is similar to FileWriteBuffer but the destination is an in-memory buffer instead of a file. - - Differences between MemoryBuffer and StringBuffer: - 1. StringBuffer has Encoding but MemoryBuffer is only a byte buffer. - 2. StringBuffer::GetString() returns a null-terminated string. MemoryBuffer::GetBuffer() returns a buffer without terminator. - - \tparam Allocator type for allocating memory buffer. - \note implements Stream concept -*/ -template -struct GenericMemoryBuffer { - typedef char Ch; // byte - - GenericMemoryBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} - - void Put(Ch c) { *stack_.template Push() = c; } - void Flush() {} - - void Clear() { stack_.Clear(); } - void ShrinkToFit() { stack_.ShrinkToFit(); } - Ch* Push(size_t count) { return stack_.template Push(count); } - void Pop(size_t count) { stack_.template Pop(count); } - - const Ch* GetBuffer() const { - return stack_.template Bottom(); - } - - size_t GetSize() const { return stack_.GetSize(); } - - static const size_t kDefaultCapacity = 256; - mutable internal::Stack stack_; -}; - -typedef GenericMemoryBuffer<> MemoryBuffer; - -//! Implement specialized version of PutN() with memset() for better performance. -template<> -inline void PutN(MemoryBuffer& memoryBuffer, char c, size_t n) { - std::memset(memoryBuffer.stack_.Push(n), c, n * sizeof(c)); -} - -} // namespace rapidjson - -#endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/tools/optimizer/rapidjson/memorystream.h b/tools/optimizer/rapidjson/memorystream.h deleted file mode 100644 index 6ed226e5a9c6f..0000000000000 --- a/tools/optimizer/rapidjson/memorystream.h +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_MEMORYSTREAM_H_ -#define RAPIDJSON_MEMORYSTREAM_H_ - -#include "rapidjson.h" - -namespace rapidjson { - -//! Represents an in-memory input byte stream. -/*! - This class is mainly for being wrapped by EncodedInputStream or AutoUTFInputStream. - - It is similar to FileReadBuffer but the source is an in-memory buffer instead of a file. - - Differences between MemoryStream and StringStream: - 1. StringStream has encoding but MemoryStream is a byte stream. - 2. MemoryStream needs size of the source buffer and the buffer don't need to be null terminated. StringStream assume null-terminated string as source. - 3. MemoryStream supports Peek4() for encoding detection. StringStream is specified with an encoding so it should not have Peek4(). - \note implements Stream concept -*/ -struct MemoryStream { - typedef char Ch; // byte - - MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} - - Ch Peek() const { return (src_ == end_) ? '\0' : *src_; } - Ch Take() { return (src_ == end_) ? '\0' : *src_++; } - size_t Tell() const { return static_cast(src_ - begin_); } - - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - - // For encoding detection only. - const Ch* Peek4() const { - return Tell() + 4 <= size_ ? src_ : 0; - } - - const Ch* src_; //!< Current read position. - const Ch* begin_; //!< Original head of the string. - const Ch* end_; //!< End of stream. - size_t size_; //!< Size of the stream. -}; - -} // namespace rapidjson - -#endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/tools/optimizer/rapidjson/msinttypes/inttypes.h b/tools/optimizer/rapidjson/msinttypes/inttypes.h deleted file mode 100644 index ac7e32b6eef30..0000000000000 --- a/tools/optimizer/rapidjson/msinttypes/inttypes.h +++ /dev/null @@ -1,306 +0,0 @@ -// ISO C9x compliant inttypes.h for Microsoft Visual Studio -// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 -// -// Copyright (c) 2006-2013 Alexander Chemeris -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the product nor the names of its contributors may -// be used to endorse or promote products derived from this software -// without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef _MSC_VER // [ -#error "Use this header only with Microsoft Visual C++ compilers!" -#endif // _MSC_VER ] - -#ifndef _MSC_INTTYPES_H_ // [ -#define _MSC_INTTYPES_H_ - -#if _MSC_VER > 1000 -#pragma once -#endif - -#include "stdint.h" - -// 7.8 Format conversion of integer types - -typedef struct { - intmax_t quot; - intmax_t rem; -} imaxdiv_t; - -// 7.8.1 Macros for format specifiers - -#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198 - -// The fprintf macros for signed integers are: -#define PRId8 "d" -#define PRIi8 "i" -#define PRIdLEAST8 "d" -#define PRIiLEAST8 "i" -#define PRIdFAST8 "d" -#define PRIiFAST8 "i" - -#define PRId16 "hd" -#define PRIi16 "hi" -#define PRIdLEAST16 "hd" -#define PRIiLEAST16 "hi" -#define PRIdFAST16 "hd" -#define PRIiFAST16 "hi" - -#define PRId32 "I32d" -#define PRIi32 "I32i" -#define PRIdLEAST32 "I32d" -#define PRIiLEAST32 "I32i" -#define PRIdFAST32 "I32d" -#define PRIiFAST32 "I32i" - -#define PRId64 "I64d" -#define PRIi64 "I64i" -#define PRIdLEAST64 "I64d" -#define PRIiLEAST64 "I64i" -#define PRIdFAST64 "I64d" -#define PRIiFAST64 "I64i" - -#define PRIdMAX "I64d" -#define PRIiMAX "I64i" - -#define PRIdPTR "Id" -#define PRIiPTR "Ii" - -// The fprintf macros for unsigned integers are: -#define PRIo8 "o" -#define PRIu8 "u" -#define PRIx8 "x" -#define PRIX8 "X" -#define PRIoLEAST8 "o" -#define PRIuLEAST8 "u" -#define PRIxLEAST8 "x" -#define PRIXLEAST8 "X" -#define PRIoFAST8 "o" -#define PRIuFAST8 "u" -#define PRIxFAST8 "x" -#define PRIXFAST8 "X" - -#define PRIo16 "ho" -#define PRIu16 "hu" -#define PRIx16 "hx" -#define PRIX16 "hX" -#define PRIoLEAST16 "ho" -#define PRIuLEAST16 "hu" -#define PRIxLEAST16 "hx" -#define PRIXLEAST16 "hX" -#define PRIoFAST16 "ho" -#define PRIuFAST16 "hu" -#define PRIxFAST16 "hx" -#define PRIXFAST16 "hX" - -#define PRIo32 "I32o" -#define PRIu32 "I32u" -#define PRIx32 "I32x" -#define PRIX32 "I32X" -#define PRIoLEAST32 "I32o" -#define PRIuLEAST32 "I32u" -#define PRIxLEAST32 "I32x" -#define PRIXLEAST32 "I32X" -#define PRIoFAST32 "I32o" -#define PRIuFAST32 "I32u" -#define PRIxFAST32 "I32x" -#define PRIXFAST32 "I32X" - -#define PRIo64 "I64o" -#define PRIu64 "I64u" -#define PRIx64 "I64x" -#define PRIX64 "I64X" -#define PRIoLEAST64 "I64o" -#define PRIuLEAST64 "I64u" -#define PRIxLEAST64 "I64x" -#define PRIXLEAST64 "I64X" -#define PRIoFAST64 "I64o" -#define PRIuFAST64 "I64u" -#define PRIxFAST64 "I64x" -#define PRIXFAST64 "I64X" - -#define PRIoMAX "I64o" -#define PRIuMAX "I64u" -#define PRIxMAX "I64x" -#define PRIXMAX "I64X" - -#define PRIoPTR "Io" -#define PRIuPTR "Iu" -#define PRIxPTR "Ix" -#define PRIXPTR "IX" - -// The fscanf macros for signed integers are: -#define SCNd8 "d" -#define SCNi8 "i" -#define SCNdLEAST8 "d" -#define SCNiLEAST8 "i" -#define SCNdFAST8 "d" -#define SCNiFAST8 "i" - -#define SCNd16 "hd" -#define SCNi16 "hi" -#define SCNdLEAST16 "hd" -#define SCNiLEAST16 "hi" -#define SCNdFAST16 "hd" -#define SCNiFAST16 "hi" - -#define SCNd32 "ld" -#define SCNi32 "li" -#define SCNdLEAST32 "ld" -#define SCNiLEAST32 "li" -#define SCNdFAST32 "ld" -#define SCNiFAST32 "li" - -#define SCNd64 "I64d" -#define SCNi64 "I64i" -#define SCNdLEAST64 "I64d" -#define SCNiLEAST64 "I64i" -#define SCNdFAST64 "I64d" -#define SCNiFAST64 "I64i" - -#define SCNdMAX "I64d" -#define SCNiMAX "I64i" - -#ifdef _WIN64 // [ -# define SCNdPTR "I64d" -# define SCNiPTR "I64i" -#else // _WIN64 ][ -# define SCNdPTR "ld" -# define SCNiPTR "li" -#endif // _WIN64 ] - -// The fscanf macros for unsigned integers are: -#define SCNo8 "o" -#define SCNu8 "u" -#define SCNx8 "x" -#define SCNX8 "X" -#define SCNoLEAST8 "o" -#define SCNuLEAST8 "u" -#define SCNxLEAST8 "x" -#define SCNXLEAST8 "X" -#define SCNoFAST8 "o" -#define SCNuFAST8 "u" -#define SCNxFAST8 "x" -#define SCNXFAST8 "X" - -#define SCNo16 "ho" -#define SCNu16 "hu" -#define SCNx16 "hx" -#define SCNX16 "hX" -#define SCNoLEAST16 "ho" -#define SCNuLEAST16 "hu" -#define SCNxLEAST16 "hx" -#define SCNXLEAST16 "hX" -#define SCNoFAST16 "ho" -#define SCNuFAST16 "hu" -#define SCNxFAST16 "hx" -#define SCNXFAST16 "hX" - -#define SCNo32 "lo" -#define SCNu32 "lu" -#define SCNx32 "lx" -#define SCNX32 "lX" -#define SCNoLEAST32 "lo" -#define SCNuLEAST32 "lu" -#define SCNxLEAST32 "lx" -#define SCNXLEAST32 "lX" -#define SCNoFAST32 "lo" -#define SCNuFAST32 "lu" -#define SCNxFAST32 "lx" -#define SCNXFAST32 "lX" - -#define SCNo64 "I64o" -#define SCNu64 "I64u" -#define SCNx64 "I64x" -#define SCNX64 "I64X" -#define SCNoLEAST64 "I64o" -#define SCNuLEAST64 "I64u" -#define SCNxLEAST64 "I64x" -#define SCNXLEAST64 "I64X" -#define SCNoFAST64 "I64o" -#define SCNuFAST64 "I64u" -#define SCNxFAST64 "I64x" -#define SCNXFAST64 "I64X" - -#define SCNoMAX "I64o" -#define SCNuMAX "I64u" -#define SCNxMAX "I64x" -#define SCNXMAX "I64X" - -#ifdef _WIN64 // [ -# define SCNoPTR "I64o" -# define SCNuPTR "I64u" -# define SCNxPTR "I64x" -# define SCNXPTR "I64X" -#else // _WIN64 ][ -# define SCNoPTR "lo" -# define SCNuPTR "lu" -# define SCNxPTR "lx" -# define SCNXPTR "lX" -#endif // _WIN64 ] - -#endif // __STDC_FORMAT_MACROS ] - -// 7.8.2 Functions for greatest-width integer types - -// 7.8.2.1 The imaxabs function -#define imaxabs _abs64 - -// 7.8.2.2 The imaxdiv function - -// This is modified version of div() function from Microsoft's div.c found -// in %MSVC.NET%\crt\src\div.c -#ifdef STATIC_IMAXDIV // [ -static -#else // STATIC_IMAXDIV ][ -_inline -#endif // STATIC_IMAXDIV ] -imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom) -{ - imaxdiv_t result; - - result.quot = numer / denom; - result.rem = numer % denom; - - if (numer < 0 && result.rem > 0) { - // did division wrong; must fix up - ++result.quot; - result.rem -= denom; - } - - return result; -} - -// 7.8.2.3 The strtoimax and strtoumax functions -#define strtoimax _strtoi64 -#define strtoumax _strtoui64 - -// 7.8.2.4 The wcstoimax and wcstoumax functions -#define wcstoimax _wcstoi64 -#define wcstoumax _wcstoui64 - - -#endif // _MSC_INTTYPES_H_ ] diff --git a/tools/optimizer/rapidjson/msinttypes/stdint.h b/tools/optimizer/rapidjson/msinttypes/stdint.h deleted file mode 100644 index bbad95af1599f..0000000000000 --- a/tools/optimizer/rapidjson/msinttypes/stdint.h +++ /dev/null @@ -1,296 +0,0 @@ -// ISO C9x compliant stdint.h for Microsoft Visual Studio -// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 -// -// Copyright (c) 2006-2013 Alexander Chemeris -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the product nor the names of its contributors may -// be used to endorse or promote products derived from this software -// without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef _MSC_VER // [ -#error "Use this header only with Microsoft Visual C++ compilers!" -#endif // _MSC_VER ] - -#ifndef _MSC_STDINT_H_ // [ -#define _MSC_STDINT_H_ - -#if _MSC_VER > 1000 -#pragma once -#endif - -// miloyip: Originally Visual Studio 2010 uses its own stdint.h. However it generates warning with INT64_C(), so change to use this file for vs2010. -#if _MSC_VER >= 1600 // [ -#include - -#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 - -#undef INT8_C -#undef INT16_C -#undef INT32_C -#undef INT64_C -#undef UINT8_C -#undef UINT16_C -#undef UINT32_C -#undef UINT64_C - -// 7.18.4.1 Macros for minimum-width integer constants - -#define INT8_C(val) val##i8 -#define INT16_C(val) val##i16 -#define INT32_C(val) val##i32 -#define INT64_C(val) val##i64 - -#define UINT8_C(val) val##ui8 -#define UINT16_C(val) val##ui16 -#define UINT32_C(val) val##ui32 -#define UINT64_C(val) val##ui64 - -// 7.18.4.2 Macros for greatest-width integer constants -// These #ifndef's are needed to prevent collisions with . -// Check out Issue 9 for the details. -#ifndef INTMAX_C // [ -# define INTMAX_C INT64_C -#endif // INTMAX_C ] -#ifndef UINTMAX_C // [ -# define UINTMAX_C UINT64_C -#endif // UINTMAX_C ] - -#endif // __STDC_CONSTANT_MACROS ] - -#else // ] _MSC_VER >= 1700 [ - -#include - -// For Visual Studio 6 in C++ mode and for many Visual Studio versions when -// compiling for ARM we should wrap include with 'extern "C++" {}' -// or compiler give many errors like this: -// error C2733: second C linkage of overloaded function 'wmemchr' not allowed -#ifdef __cplusplus -extern "C" { -#endif -# include -#ifdef __cplusplus -} -#endif - -// Define _W64 macros to mark types changing their size, like intptr_t. -#ifndef _W64 -# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 -# define _W64 __w64 -# else -# define _W64 -# endif -#endif - - -// 7.18.1 Integer types - -// 7.18.1.1 Exact-width integer types - -// Visual Studio 6 and Embedded Visual C++ 4 doesn't -// realize that, e.g. char has the same size as __int8 -// so we give up on __intX for them. -#if (_MSC_VER < 1300) - typedef signed char int8_t; - typedef signed short int16_t; - typedef signed int int32_t; - typedef unsigned char uint8_t; - typedef unsigned short uint16_t; - typedef unsigned int uint32_t; -#else - typedef signed __int8 int8_t; - typedef signed __int16 int16_t; - typedef signed __int32 int32_t; - typedef unsigned __int8 uint8_t; - typedef unsigned __int16 uint16_t; - typedef unsigned __int32 uint32_t; -#endif -typedef signed __int64 int64_t; -typedef unsigned __int64 uint64_t; - - -// 7.18.1.2 Minimum-width integer types -typedef int8_t int_least8_t; -typedef int16_t int_least16_t; -typedef int32_t int_least32_t; -typedef int64_t int_least64_t; -typedef uint8_t uint_least8_t; -typedef uint16_t uint_least16_t; -typedef uint32_t uint_least32_t; -typedef uint64_t uint_least64_t; - -// 7.18.1.3 Fastest minimum-width integer types -typedef int8_t int_fast8_t; -typedef int16_t int_fast16_t; -typedef int32_t int_fast32_t; -typedef int64_t int_fast64_t; -typedef uint8_t uint_fast8_t; -typedef uint16_t uint_fast16_t; -typedef uint32_t uint_fast32_t; -typedef uint64_t uint_fast64_t; - -// 7.18.1.4 Integer types capable of holding object pointers -#ifdef _WIN64 // [ - typedef signed __int64 intptr_t; - typedef unsigned __int64 uintptr_t; -#else // _WIN64 ][ - typedef _W64 signed int intptr_t; - typedef _W64 unsigned int uintptr_t; -#endif // _WIN64 ] - -// 7.18.1.5 Greatest-width integer types -typedef int64_t intmax_t; -typedef uint64_t uintmax_t; - - -// 7.18.2 Limits of specified-width integer types - -#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 - -// 7.18.2.1 Limits of exact-width integer types -#define INT8_MIN ((int8_t)_I8_MIN) -#define INT8_MAX _I8_MAX -#define INT16_MIN ((int16_t)_I16_MIN) -#define INT16_MAX _I16_MAX -#define INT32_MIN ((int32_t)_I32_MIN) -#define INT32_MAX _I32_MAX -#define INT64_MIN ((int64_t)_I64_MIN) -#define INT64_MAX _I64_MAX -#define UINT8_MAX _UI8_MAX -#define UINT16_MAX _UI16_MAX -#define UINT32_MAX _UI32_MAX -#define UINT64_MAX _UI64_MAX - -// 7.18.2.2 Limits of minimum-width integer types -#define INT_LEAST8_MIN INT8_MIN -#define INT_LEAST8_MAX INT8_MAX -#define INT_LEAST16_MIN INT16_MIN -#define INT_LEAST16_MAX INT16_MAX -#define INT_LEAST32_MIN INT32_MIN -#define INT_LEAST32_MAX INT32_MAX -#define INT_LEAST64_MIN INT64_MIN -#define INT_LEAST64_MAX INT64_MAX -#define UINT_LEAST8_MAX UINT8_MAX -#define UINT_LEAST16_MAX UINT16_MAX -#define UINT_LEAST32_MAX UINT32_MAX -#define UINT_LEAST64_MAX UINT64_MAX - -// 7.18.2.3 Limits of fastest minimum-width integer types -#define INT_FAST8_MIN INT8_MIN -#define INT_FAST8_MAX INT8_MAX -#define INT_FAST16_MIN INT16_MIN -#define INT_FAST16_MAX INT16_MAX -#define INT_FAST32_MIN INT32_MIN -#define INT_FAST32_MAX INT32_MAX -#define INT_FAST64_MIN INT64_MIN -#define INT_FAST64_MAX INT64_MAX -#define UINT_FAST8_MAX UINT8_MAX -#define UINT_FAST16_MAX UINT16_MAX -#define UINT_FAST32_MAX UINT32_MAX -#define UINT_FAST64_MAX UINT64_MAX - -// 7.18.2.4 Limits of integer types capable of holding object pointers -#ifdef _WIN64 // [ -# define INTPTR_MIN INT64_MIN -# define INTPTR_MAX INT64_MAX -# define UINTPTR_MAX UINT64_MAX -#else // _WIN64 ][ -# define INTPTR_MIN INT32_MIN -# define INTPTR_MAX INT32_MAX -# define UINTPTR_MAX UINT32_MAX -#endif // _WIN64 ] - -// 7.18.2.5 Limits of greatest-width integer types -#define INTMAX_MIN INT64_MIN -#define INTMAX_MAX INT64_MAX -#define UINTMAX_MAX UINT64_MAX - -// 7.18.3 Limits of other integer types - -#ifdef _WIN64 // [ -# define PTRDIFF_MIN _I64_MIN -# define PTRDIFF_MAX _I64_MAX -#else // _WIN64 ][ -# define PTRDIFF_MIN _I32_MIN -# define PTRDIFF_MAX _I32_MAX -#endif // _WIN64 ] - -#define SIG_ATOMIC_MIN INT_MIN -#define SIG_ATOMIC_MAX INT_MAX - -#ifndef SIZE_MAX // [ -# ifdef _WIN64 // [ -# define SIZE_MAX _UI64_MAX -# else // _WIN64 ][ -# define SIZE_MAX _UI32_MAX -# endif // _WIN64 ] -#endif // SIZE_MAX ] - -// WCHAR_MIN and WCHAR_MAX are also defined in -#ifndef WCHAR_MIN // [ -# define WCHAR_MIN 0 -#endif // WCHAR_MIN ] -#ifndef WCHAR_MAX // [ -# define WCHAR_MAX _UI16_MAX -#endif // WCHAR_MAX ] - -#define WINT_MIN 0 -#define WINT_MAX _UI16_MAX - -#endif // __STDC_LIMIT_MACROS ] - - -// 7.18.4 Limits of other integer types - -#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 - -// 7.18.4.1 Macros for minimum-width integer constants - -#define INT8_C(val) val##i8 -#define INT16_C(val) val##i16 -#define INT32_C(val) val##i32 -#define INT64_C(val) val##i64 - -#define UINT8_C(val) val##ui8 -#define UINT16_C(val) val##ui16 -#define UINT32_C(val) val##ui32 -#define UINT64_C(val) val##ui64 - -// 7.18.4.2 Macros for greatest-width integer constants -// These #ifndef's are needed to prevent collisions with . -// Check out Issue 9 for the details. -#ifndef INTMAX_C // [ -# define INTMAX_C INT64_C -#endif // INTMAX_C ] -#ifndef UINTMAX_C // [ -# define UINTMAX_C UINT64_C -#endif // UINTMAX_C ] - -#endif // __STDC_CONSTANT_MACROS ] - -#endif // _MSC_VER >= 1600 ] - -#endif // _MSC_STDINT_H_ ] diff --git a/tools/optimizer/rapidjson/prettywriter.h b/tools/optimizer/rapidjson/prettywriter.h deleted file mode 100644 index 4eac8d76f8309..0000000000000 --- a/tools/optimizer/rapidjson/prettywriter.h +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_PRETTYWRITER_H_ -#define RAPIDJSON_PRETTYWRITER_H_ - -#include "writer.h" - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#endif - -namespace rapidjson { - -//! Writer with indentation and spacing. -/*! - \tparam OutputStream Type of ouptut os. - \tparam SourceEncoding Encoding of source string. - \tparam TargetEncoding Encoding of output stream. - \tparam StackAllocator Type of allocator for allocating memory of stack. -*/ -template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator> -class PrettyWriter : public Writer { -public: - typedef Writer Base; - typedef typename Base::Ch Ch; - - //! Constructor - /*! \param os Output stream. - \param allocator User supplied allocator. If it is null, it will create a private one. - \param levelDepth Initial capacity of stack. - */ - PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : - Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} - - //! Set custom indentation. - /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). - \param indentCharCount Number of indent characters for each indentation level. - \note The default indentation is 4 spaces. - */ - PrettyWriter& SetIndent(Ch indentChar, unsigned indentCharCount) { - RAPIDJSON_ASSERT(indentChar == ' ' || indentChar == '\t' || indentChar == '\n' || indentChar == '\r'); - indentChar_ = indentChar; - indentCharCount_ = indentCharCount; - return *this; - } - - /*! @name Implementation of Handler - \see Handler - */ - //@{ - - bool Null() { PrettyPrefix(kNullType); return Base::WriteNull(); } - bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::WriteBool(b); } - bool Int(int i) { PrettyPrefix(kNumberType); return Base::WriteInt(i); } - bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::WriteUint(u); } - bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::WriteInt64(i64); } - bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); } - bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); } - - bool String(const Ch* str, SizeType length, bool copy = false) { - (void)copy; - PrettyPrefix(kStringType); - return Base::WriteString(str, length); - } - - bool StartObject() { - PrettyPrefix(kObjectType); - new (Base::level_stack_.template Push()) typename Base::Level(false); - return Base::WriteStartObject(); - } - - bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } - - bool EndObject(SizeType memberCount = 0) { - (void)memberCount; - RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); - RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); - bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; - - if (!empty) { - Base::os_->Put('\n'); - WriteIndent(); - } - if (!Base::WriteEndObject()) - return false; - if (Base::level_stack_.Empty()) // end of json text - Base::os_->Flush(); - return true; - } - - bool StartArray() { - PrettyPrefix(kArrayType); - new (Base::level_stack_.template Push()) typename Base::Level(true); - return Base::WriteStartArray(); - } - - bool EndArray(SizeType memberCount = 0) { - (void)memberCount; - RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); - RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); - bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; - - if (!empty) { - Base::os_->Put('\n'); - WriteIndent(); - } - if (!Base::WriteEndArray()) - return false; - if (Base::level_stack_.Empty()) // end of json text - Base::os_->Flush(); - return true; - } - - //@} - - /*! @name Convenience extensions */ - //@{ - - //! Simpler but slower overload. - bool String(const Ch* str) { return String(str, internal::StrLen(str)); } - bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } - - //@} -protected: - void PrettyPrefix(Type type) { - (void)type; - if (Base::level_stack_.GetSize() != 0) { // this value is not at root - typename Base::Level* level = Base::level_stack_.template Top(); - - if (level->inArray) { - if (level->valueCount > 0) { - Base::os_->Put(','); // add comma if it is not the first element in array - Base::os_->Put('\n'); - } - else - Base::os_->Put('\n'); - WriteIndent(); - } - else { // in object - if (level->valueCount > 0) { - if (level->valueCount % 2 == 0) { - Base::os_->Put(','); - Base::os_->Put('\n'); - } - else { - Base::os_->Put(':'); - Base::os_->Put(' '); - } - } - else - Base::os_->Put('\n'); - - if (level->valueCount % 2 == 0) - WriteIndent(); - } - if (!level->inArray && level->valueCount % 2 == 0) - RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name - level->valueCount++; - } - else { - RAPIDJSON_ASSERT(!Base::hasRoot_); // Should only has one and only one root. - Base::hasRoot_ = true; - } - } - - void WriteIndent() { - size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; - PutN(*Base::os_, indentChar_, count); - } - - Ch indentChar_; - unsigned indentCharCount_; - -private: - // Prohibit copy constructor & assignment operator. - PrettyWriter(const PrettyWriter&); - PrettyWriter& operator=(const PrettyWriter&); -}; - -} // namespace rapidjson - -#ifdef __GNUC__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/tools/optimizer/rapidjson/rapidjson.h b/tools/optimizer/rapidjson/rapidjson.h deleted file mode 100644 index 3f743234ddd55..0000000000000 --- a/tools/optimizer/rapidjson/rapidjson.h +++ /dev/null @@ -1,567 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_RAPIDJSON_H_ -#define RAPIDJSON_RAPIDJSON_H_ - -// Copyright (c) 2011 Milo Yip (miloyip@gmail.com) -// Version 0.1 - -/*!\file rapidjson.h - \brief common definitions and configuration - - \see RAPIDJSON_CONFIG - */ - -/*! \defgroup RAPIDJSON_CONFIG RapidJSON configuration - \brief Configuration macros for library features - - Some RapidJSON features are configurable to adapt the library to a wide - variety of platforms, environments and usage scenarios. Most of the - features can be configured in terms of overriden or predefined - preprocessor macros at compile-time. - - Some additional customization is available in the \ref RAPIDJSON_ERRORS APIs. - - \note These macros should be given on the compiler command-line - (where applicable) to avoid inconsistent values when compiling - different translation units of a single application. - */ - -#include // malloc(), realloc(), free(), size_t -#include // memset(), memcpy(), memmove(), memcmp() - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_NO_INT64DEFINE - -/*! \def RAPIDJSON_NO_INT64DEFINE - \ingroup RAPIDJSON_CONFIG - \brief Use external 64-bit integer types. - - RapidJSON requires the 64-bit integer types \c int64_t and \c uint64_t types - to be available at global scope. - - If users have their own definition, define RAPIDJSON_NO_INT64DEFINE to - prevent RapidJSON from defining its own types. -*/ -#ifndef RAPIDJSON_NO_INT64DEFINE -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#ifdef _MSC_VER -#include "msinttypes/stdint.h" -#include "msinttypes/inttypes.h" -#else -// Other compilers should have this. -#include -#include -#endif -//!@endcond -#ifdef RAPIDJSON_DOXYGEN_RUNNING -#define RAPIDJSON_NO_INT64DEFINE -#endif -#endif // RAPIDJSON_NO_INT64TYPEDEF - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_FORCEINLINE - -#ifndef RAPIDJSON_FORCEINLINE -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#ifdef _MSC_VER -#define RAPIDJSON_FORCEINLINE __forceinline -#elif defined(__GNUC__) && __GNUC__ >= 4 -#define RAPIDJSON_FORCEINLINE __attribute__((always_inline)) -#else -#define RAPIDJSON_FORCEINLINE -#endif -//!@endcond -#endif // RAPIDJSON_FORCEINLINE - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_ENDIAN -#define RAPIDJSON_LITTLEENDIAN 0 //!< Little endian machine -#define RAPIDJSON_BIGENDIAN 1 //!< Big endian machine - -//! Endianness of the machine. -/*! - \def RAPIDJSON_ENDIAN - \ingroup RAPIDJSON_CONFIG - - GCC 4.6 provided macro for detecting endianness of the target machine. But other - compilers may not have this. User can define RAPIDJSON_ENDIAN to either - \ref RAPIDJSON_LITTLEENDIAN or \ref RAPIDJSON_BIGENDIAN. - - Default detection implemented with reference to - \li https://gcc.gnu.org/onlinedocs/gcc-4.6.0/cpp/Common-Predefined-Macros.html - \li http://www.boost.org/doc/libs/1_42_0/boost/detail/endian.hpp -*/ -#ifndef RAPIDJSON_ENDIAN -// Detect with GCC 4.6's macro -# ifdef __BYTE_ORDER__ -# if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN -# elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN -# else -# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. -# endif // __BYTE_ORDER__ -// Detect with GLIBC's endian.h -# elif defined(__GLIBC__) -# include -# if (__BYTE_ORDER == __LITTLE_ENDIAN) -# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN -# elif (__BYTE_ORDER == __BIG_ENDIAN) -# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN -# else -# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. -# endif // __GLIBC__ -// Detect with _LITTLE_ENDIAN and _BIG_ENDIAN macro -# elif defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN) -# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN -# elif defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN) -# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN -// Detect with architecture macros -# elif defined(__sparc) || defined(__sparc__) || defined(_POWER) || defined(__powerpc__) || defined(__ppc__) || defined(__hpux) || defined(__hppa) || defined(_MIPSEB) || defined(_POWER) || defined(__s390__) -# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN -# elif defined(__i386__) || defined(__alpha__) || defined(__ia64) || defined(__ia64__) || defined(_M_IX86) || defined(_M_IA64) || defined(_M_ALPHA) || defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) || defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || defined(__bfin__) -# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN -# elif defined(RAPIDJSON_DOXYGEN_RUNNING) -# define RAPIDJSON_ENDIAN -# else -# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. -# endif -#endif // RAPIDJSON_ENDIAN - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_64BIT - -//! Whether using 64-bit architecture -#ifndef RAPIDJSON_64BIT -#if defined(__LP64__) || defined(_WIN64) -#define RAPIDJSON_64BIT 1 -#else -#define RAPIDJSON_64BIT 0 -#endif -#endif // RAPIDJSON_64BIT - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_ALIGN - -//! Data alignment of the machine. -/*! \ingroup RAPIDJSON_CONFIG - \param x pointer to align - - Some machines require strict data alignment. Currently the default uses 4 bytes - alignment. User can customize by defining the RAPIDJSON_ALIGN function macro., -*/ -#ifndef RAPIDJSON_ALIGN -#define RAPIDJSON_ALIGN(x) ((x + 3u) & ~3u) -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_UINT64_C2 - -//! Construct a 64-bit literal by a pair of 32-bit integer. -/*! - 64-bit literal with or without ULL suffix is prone to compiler warnings. - UINT64_C() is C macro which cause compilation problems. - Use this macro to define 64-bit constants by a pair of 32-bit integer. -*/ -#ifndef RAPIDJSON_UINT64_C2 -#define RAPIDJSON_UINT64_C2(high32, low32) ((static_cast(high32) << 32) | static_cast(low32)) -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_SIMD - -/*! \def RAPIDJSON_SIMD - \ingroup RAPIDJSON_CONFIG - \brief Enable SSE2/SSE4.2 optimization. - - RapidJSON supports optimized implementations for some parsing operations - based on the SSE2 or SSE4.2 SIMD extensions on modern Intel-compatible - processors. - - To enable these optimizations, two different symbols can be defined; - \code - // Enable SSE2 optimization. - #define RAPIDJSON_SSE2 - - // Enable SSE4.2 optimization. - #define RAPIDJSON_SSE42 - \endcode - - \c RAPIDJSON_SSE42 takes precedence, if both are defined. - - If any of these symbols is defined, RapidJSON defines the macro - \c RAPIDJSON_SIMD to indicate the availability of the optimized code. -*/ -#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) \ - || defined(RAPIDJSON_DOXYGEN_RUNNING) -#define RAPIDJSON_SIMD -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_NO_SIZETYPEDEFINE - -#ifndef RAPIDJSON_NO_SIZETYPEDEFINE -/*! \def RAPIDJSON_NO_SIZETYPEDEFINE - \ingroup RAPIDJSON_CONFIG - \brief User-provided \c SizeType definition. - - In order to avoid using 32-bit size types for indexing strings and arrays, - define this preprocessor symbol and provide the type rapidjson::SizeType - before including RapidJSON: - \code - #define RAPIDJSON_NO_SIZETYPEDEFINE - namespace rapidjson { typedef ::std::size_t SizeType; } - #include "rapidjson/..." - \endcode - - \see rapidjson::SizeType -*/ -#ifdef RAPIDJSON_DOXYGEN_RUNNING -#define RAPIDJSON_NO_SIZETYPEDEFINE -#endif -namespace rapidjson { -//! Size type (for string lengths, array sizes, etc.) -/*! RapidJSON uses 32-bit array/string indices even on 64-bit platforms, - instead of using \c size_t. Users may override the SizeType by defining - \ref RAPIDJSON_NO_SIZETYPEDEFINE. -*/ -typedef unsigned SizeType; -} // namespace rapidjson -#endif - -// always import std::size_t to rapidjson namespace -namespace rapidjson { -using std::size_t; -} // namespace rapidjson - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_ASSERT - -//! Assertion. -/*! \ingroup RAPIDJSON_CONFIG - By default, rapidjson uses C \c assert() for internal assertions. - User can override it by defining RAPIDJSON_ASSERT(x) macro. - - \note Parsing errors are handled and can be customized by the - \ref RAPIDJSON_ERRORS APIs. -*/ -#ifndef RAPIDJSON_ASSERT -#include -#define RAPIDJSON_ASSERT(x) assert(x) -#endif // RAPIDJSON_ASSERT - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_STATIC_ASSERT - -// Adopt from boost -#ifndef RAPIDJSON_STATIC_ASSERT -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -namespace rapidjson { - -template struct STATIC_ASSERTION_FAILURE; -template <> struct STATIC_ASSERTION_FAILURE { enum { value = 1 }; }; -template struct StaticAssertTest {}; -} // namespace rapidjson - -#define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) -#define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) -#define RAPIDJSON_DO_JOIN2(X, Y) X##Y - -#if defined(__GNUC__) -#define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE __attribute__((unused)) -#else -#define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE -#endif -//!@endcond - -/*! \def RAPIDJSON_STATIC_ASSERT - \brief (Internal) macro to check for conditions at compile-time - \param x compile-time condition - \hideinitializer - */ -#define RAPIDJSON_STATIC_ASSERT(x) typedef ::rapidjson::StaticAssertTest<\ - sizeof(::rapidjson::STATIC_ASSERTION_FAILURE)>\ - RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE -#endif - -/////////////////////////////////////////////////////////////////////////////// -// Helpers - -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN - -#define RAPIDJSON_MULTILINEMACRO_BEGIN do { -#define RAPIDJSON_MULTILINEMACRO_END \ -} while((void)0, 0) - -// adopted from Boost -#define RAPIDJSON_VERSION_CODE(x,y,z) \ - (((x)*100000) + ((y)*100) + (z)) - -// token stringification -#define RAPIDJSON_STRINGIFY(x) RAPIDJSON_DO_STRINGIFY(x) -#define RAPIDJSON_DO_STRINGIFY(x) #x - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_DIAG_PUSH/POP, RAPIDJSON_DIAG_OFF - -#if defined(__GNUC__) -#define RAPIDJSON_GNUC \ - RAPIDJSON_VERSION_CODE(__GNUC__,__GNUC_MINOR__,__GNUC_PATCHLEVEL__) -#endif - -#if defined(__clang__) || (defined(RAPIDJSON_GNUC) && RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,2,0)) - -#define RAPIDJSON_PRAGMA(x) _Pragma(RAPIDJSON_STRINGIFY(x)) -#define RAPIDJSON_DIAG_PRAGMA(x) RAPIDJSON_PRAGMA(GCC diagnostic x) -#define RAPIDJSON_DIAG_OFF(x) \ - RAPIDJSON_DIAG_PRAGMA(ignored RAPIDJSON_STRINGIFY(RAPIDJSON_JOIN(-W,x))) - -// push/pop support in Clang and GCC>=4.6 -#if defined(__clang__) || (defined(RAPIDJSON_GNUC) && RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) -#define RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_PRAGMA(push) -#define RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_PRAGMA(pop) -#else // GCC >= 4.2, < 4.6 -#define RAPIDJSON_DIAG_PUSH /* ignored */ -#define RAPIDJSON_DIAG_POP /* ignored */ -#endif - -#elif defined(_MSC_VER) - -// pragma (MSVC specific) -#define RAPIDJSON_PRAGMA(x) __pragma(x) -#define RAPIDJSON_DIAG_PRAGMA(x) RAPIDJSON_PRAGMA(warning(x)) - -#define RAPIDJSON_DIAG_OFF(x) RAPIDJSON_DIAG_PRAGMA(disable: x) -#define RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_PRAGMA(push) -#define RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_PRAGMA(pop) - -#else - -#define RAPIDJSON_DIAG_OFF(x) /* ignored */ -#define RAPIDJSON_DIAG_PUSH /* ignored */ -#define RAPIDJSON_DIAG_POP /* ignored */ - -#endif // RAPIDJSON_DIAG_* - -/////////////////////////////////////////////////////////////////////////////// -// C++11 features - -#ifndef RAPIDJSON_HAS_CXX11_RVALUE_REFS -#if defined(__clang__) -#define RAPIDJSON_HAS_CXX11_RVALUE_REFS __has_feature(cxx_rvalue_references) -#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ - (defined(_MSC_VER) && _MSC_VER >= 1600) - -#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 -#else -#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 -#endif -#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS - -#ifndef RAPIDJSON_HAS_CXX11_NOEXCEPT -#if defined(__clang__) -#define RAPIDJSON_HAS_CXX11_NOEXCEPT __has_feature(cxx_noexcept) -#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) -// (defined(_MSC_VER) && _MSC_VER >= ????) // not yet supported -#define RAPIDJSON_HAS_CXX11_NOEXCEPT 1 -#else -#define RAPIDJSON_HAS_CXX11_NOEXCEPT 0 -#endif -#endif -#if RAPIDJSON_HAS_CXX11_NOEXCEPT -#define RAPIDJSON_NOEXCEPT noexcept -#else -#define RAPIDJSON_NOEXCEPT /* noexcept */ -#endif // RAPIDJSON_HAS_CXX11_NOEXCEPT - -// no automatic detection, yet -#ifndef RAPIDJSON_HAS_CXX11_TYPETRAITS -#define RAPIDJSON_HAS_CXX11_TYPETRAITS 0 -#endif - -//!@endcond - -/////////////////////////////////////////////////////////////////////////////// -// Allocators and Encodings - -#include "allocators.h" -#include "encodings.h" - -//! main RapidJSON namespace -namespace rapidjson { - -/////////////////////////////////////////////////////////////////////////////// -// Stream - -/*! \class rapidjson::Stream - \brief Concept for reading and writing characters. - - For read-only stream, no need to implement PutBegin(), Put(), Flush() and PutEnd(). - - For write-only stream, only need to implement Put() and Flush(). - -\code -concept Stream { - typename Ch; //!< Character type of the stream. - - //! Read the current character from stream without moving the read cursor. - Ch Peek() const; - - //! Read the current character from stream and moving the read cursor to next character. - Ch Take(); - - //! Get the current read cursor. - //! \return Number of characters read from start. - size_t Tell(); - - //! Begin writing operation at the current read pointer. - //! \return The begin writer pointer. - Ch* PutBegin(); - - //! Write a character. - void Put(Ch c); - - //! Flush the buffer. - void Flush(); - - //! End the writing operation. - //! \param begin The begin write pointer returned by PutBegin(). - //! \return Number of characters written. - size_t PutEnd(Ch* begin); -} -\endcode -*/ - -//! Provides additional information for stream. -/*! - By using traits pattern, this type provides a default configuration for stream. - For custom stream, this type can be specialized for other configuration. - See TEST(Reader, CustomStringStream) in readertest.cpp for example. -*/ -template -struct StreamTraits { - //! Whether to make local copy of stream for optimization during parsing. - /*! - By default, for safety, streams do not use local copy optimization. - Stream that can be copied fast should specialize this, like StreamTraits. - */ - enum { copyOptimization = 0 }; -}; - -//! Put N copies of a character to a stream. -template -inline void PutN(Stream& stream, Ch c, size_t n) { - for (size_t i = 0; i < n; i++) - stream.Put(c); -} - -/////////////////////////////////////////////////////////////////////////////// -// StringStream - -//! Read-only string stream. -/*! \note implements Stream concept -*/ -template -struct GenericStringStream { - typedef typename Encoding::Ch Ch; - - GenericStringStream(const Ch *src) : src_(src), head_(src) {} - - Ch Peek() const { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() const { return static_cast(src_ - head_); } - - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - - const Ch* src_; //!< Current read position. - const Ch* head_; //!< Original head of the string. -}; - -template -struct StreamTraits > { - enum { copyOptimization = 1 }; -}; - -//! String stream with UTF8 encoding. -typedef GenericStringStream > StringStream; - -/////////////////////////////////////////////////////////////////////////////// -// InsituStringStream - -//! A read-write string stream. -/*! This string stream is particularly designed for in-situ parsing. - \note implements Stream concept -*/ -template -struct GenericInsituStringStream { - typedef typename Encoding::Ch Ch; - - GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} - - // Read - Ch Peek() { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() { return static_cast(src_ - head_); } - - // Write - void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } - - Ch* PutBegin() { return dst_ = src_; } - size_t PutEnd(Ch* begin) { return static_cast(dst_ - begin); } - void Flush() {} - - Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } - void Pop(size_t count) { dst_ -= count; } - - Ch* src_; - Ch* dst_; - Ch* head_; -}; - -template -struct StreamTraits > { - enum { copyOptimization = 1 }; -}; - -//! Insitu string stream with UTF8 encoding. -typedef GenericInsituStringStream > InsituStringStream; - -/////////////////////////////////////////////////////////////////////////////// -// Type - -//! Type of JSON value -enum Type { - kNullType = 0, //!< null - kFalseType = 1, //!< false - kTrueType = 2, //!< true - kObjectType = 3, //!< object - kArrayType = 4, //!< array - kStringType = 5, //!< string - kNumberType = 6 //!< number -}; - -} // namespace rapidjson - -#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/tools/optimizer/rapidjson/reader.h b/tools/optimizer/rapidjson/reader.h deleted file mode 100644 index f41ba2fd7b82a..0000000000000 --- a/tools/optimizer/rapidjson/reader.h +++ /dev/null @@ -1,1369 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_READER_H_ -#define RAPIDJSON_READER_H_ - -/*! \file reader.h */ - -#include "rapidjson.h" -#include "encodings.h" -#include "internal/meta.h" -#include "internal/pow10.h" -#include "internal/stack.h" - -#if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) -#include -#pragma intrinsic(_BitScanForward) -#endif -#ifdef RAPIDJSON_SSE42 -#include -#elif defined(RAPIDJSON_SSE2) -#include -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant -RAPIDJSON_DIAG_OFF(4702) // unreachable code -#endif - -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#define RAPIDJSON_NOTHING /* deliberately empty */ -#ifndef RAPIDJSON_PARSE_ERROR_EARLY_RETURN -#define RAPIDJSON_PARSE_ERROR_EARLY_RETURN(value) \ - RAPIDJSON_MULTILINEMACRO_BEGIN \ - if (HasParseError()) { return value; } \ - RAPIDJSON_MULTILINEMACRO_END -#endif -#define RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID \ - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(RAPIDJSON_NOTHING) -//!@endcond - -/*! \def RAPIDJSON_PARSE_ERROR_NORETURN - \ingroup RAPIDJSON_ERRORS - \brief Macro to indicate a parse error. - \param parseErrorCode \ref rapidjson::ParseErrorCode of the error - \param offset position of the error in JSON input (\c size_t) - - This macros can be used as a customization point for the internal - error handling mechanism of RapidJSON. - - A common usage model is to throw an exception instead of requiring the - caller to explicitly check the \ref rapidjson::GenericReader::Parse's - return value: - - \code - #define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode,offset) \ - throw ParseException(parseErrorCode, #parseErrorCode, offset) - - #include // std::runtime_error - #include "rapidjson/error/error.h" // rapidjson::ParseResult - - struct ParseException : std::runtime_error, rapidjson::ParseResult { - ParseException(rapidjson::ParseErrorCode code, const char* msg, size_t offset) - : std::runtime_error(msg), ParseResult(code, offset) {} - }; - - #include "rapidjson/reader.h" - \endcode - - \see RAPIDJSON_PARSE_ERROR, rapidjson::GenericReader::Parse - */ -#ifndef RAPIDJSON_PARSE_ERROR_NORETURN -#define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset) \ - RAPIDJSON_MULTILINEMACRO_BEGIN \ - RAPIDJSON_ASSERT(!HasParseError()); /* Error can only be assigned once */ \ - SetParseError(parseErrorCode, offset); \ - RAPIDJSON_MULTILINEMACRO_END -#endif - -/*! \def RAPIDJSON_PARSE_ERROR - \ingroup RAPIDJSON_ERRORS - \brief (Internal) macro to indicate and handle a parse error. - \param parseErrorCode \ref rapidjson::ParseErrorCode of the error - \param offset position of the error in JSON input (\c size_t) - - Invokes RAPIDJSON_PARSE_ERROR_NORETURN and stops the parsing. - - \see RAPIDJSON_PARSE_ERROR_NORETURN - \hideinitializer - */ -#ifndef RAPIDJSON_PARSE_ERROR -#define RAPIDJSON_PARSE_ERROR(parseErrorCode, offset) \ - RAPIDJSON_MULTILINEMACRO_BEGIN \ - RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset); \ - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; \ - RAPIDJSON_MULTILINEMACRO_END -#endif - -#include "error/error.h" // ParseErrorCode, ParseResult - -namespace rapidjson { - -/////////////////////////////////////////////////////////////////////////////// -// ParseFlag - -//! Combination of parseFlags -/*! \see Reader::Parse, Document::Parse, Document::ParseInsitu, Document::ParseStream - */ -enum ParseFlag { - kParseDefaultFlags = 0, //!< Default parse flags. Non-destructive parsing. Text strings are decoded into allocated buffer. - kParseInsituFlag = 1, //!< In-situ(destructive) parsing. - kParseValidateEncodingFlag = 2, //!< Validate encoding of JSON strings. - kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing. - kParseStopWhenDoneFlag = 8 //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. -}; - -/////////////////////////////////////////////////////////////////////////////// -// Handler - -/*! \class rapidjson::Handler - \brief Concept for receiving events from GenericReader upon parsing. - The functions return true if no error occurs. If they return false, - the event publisher should terminate the process. -\code -concept Handler { - typename Ch; - - bool Null(); - bool Bool(bool b); - bool Int(int i); - bool Uint(unsigned i); - bool Int64(int64_t i); - bool Uint64(uint64_t i); - bool Double(double d); - bool String(const Ch* str, SizeType length, bool copy); - bool StartObject(); - bool Key(const Ch* str, SizeType length, bool copy); - bool EndObject(SizeType memberCount); - bool StartArray(); - bool EndArray(SizeType elementCount); -}; -\endcode -*/ -/////////////////////////////////////////////////////////////////////////////// -// BaseReaderHandler - -//! Default implementation of Handler. -/*! This can be used as base class of any reader handler. - \note implements Handler concept -*/ -template, typename Derived = void> -struct BaseReaderHandler { - typedef typename Encoding::Ch Ch; - - typedef typename internal::SelectIf, BaseReaderHandler, Derived>::Type Override; - - bool Default() { return true; } - bool Null() { return static_cast(*this).Default(); } - bool Bool(bool) { return static_cast(*this).Default(); } - bool Int(int) { return static_cast(*this).Default(); } - bool Uint(unsigned) { return static_cast(*this).Default(); } - bool Int64(int64_t) { return static_cast(*this).Default(); } - bool Uint64(uint64_t) { return static_cast(*this).Default(); } - bool Double(double) { return static_cast(*this).Default(); } - bool String(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } - bool StartObject() { return static_cast(*this).Default(); } - bool Key(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } - bool EndObject(SizeType) { return static_cast(*this).Default(); } - bool StartArray() { return static_cast(*this).Default(); } - bool EndArray(SizeType) { return static_cast(*this).Default(); } -}; - -/////////////////////////////////////////////////////////////////////////////// -// StreamLocalCopy - -namespace internal { - -template::copyOptimization> -class StreamLocalCopy; - -//! Do copy optimization. -template -class StreamLocalCopy { -public: - StreamLocalCopy(Stream& original) : s(original), original_(original) {} - ~StreamLocalCopy() { original_ = s; } - - Stream s; - -private: - StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; - - Stream& original_; -}; - -//! Keep reference. -template -class StreamLocalCopy { -public: - StreamLocalCopy(Stream& original) : s(original) {} - - Stream& s; - -private: - StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; -}; - -} // namespace internal - -/////////////////////////////////////////////////////////////////////////////// -// SkipWhitespace - -//! Skip the JSON white spaces in a stream. -/*! \param is A input stream for skipping white spaces. - \note This function has SSE2/SSE4.2 specialization. -*/ -template -void SkipWhitespace(InputStream& is) { - internal::StreamLocalCopy copy(is); - InputStream& s(copy.s); - - while (s.Peek() == ' ' || s.Peek() == '\n' || s.Peek() == '\r' || s.Peek() == '\t') - s.Take(); -} - -#ifdef RAPIDJSON_SSE42 -//! Skip whitespace with SSE 4.2 pcmpistrm instruction, testing 16 8-byte characters at once. -inline const char *SkipWhitespace_SIMD(const char* p) { - // Fast return for single non-whitespace - if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') - ++p; - else - return p; - - // 16-byte align to the next boundary - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & ~15); - while (p != nextAligned) - if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') - ++p; - else - return p; - - // The rest of string using SIMD - static const char whitespace[16] = " \n\r\t"; - const __m128i w = _mm_loadu_si128((const __m128i *)&whitespace[0]); - - for (;; p += 16) { - const __m128i s = _mm_load_si128((const __m128i *)p); - const unsigned r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); - if (r != 0) { // some of characters is non-whitespace -#ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; -#else - return p + __builtin_ffs(r) - 1; -#endif - } - } -} - -#elif defined(RAPIDJSON_SSE2) - -//! Skip whitespace with SSE2 instructions, testing 16 8-byte characters at once. -inline const char *SkipWhitespace_SIMD(const char* p) { - // Fast return for single non-whitespace - if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') - ++p; - else - return p; - - // 16-byte align to the next boundary - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & ~15); - while (p != nextAligned) - if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') - ++p; - else - return p; - - // The rest of string - static const char whitespaces[4][17] = { - " ", - "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", - "\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r", - "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"}; - - const __m128i w0 = _mm_loadu_si128((const __m128i *)&whitespaces[0][0]); - const __m128i w1 = _mm_loadu_si128((const __m128i *)&whitespaces[1][0]); - const __m128i w2 = _mm_loadu_si128((const __m128i *)&whitespaces[2][0]); - const __m128i w3 = _mm_loadu_si128((const __m128i *)&whitespaces[3][0]); - - for (;; p += 16) { - const __m128i s = _mm_load_si128((const __m128i *)p); - __m128i x = _mm_cmpeq_epi8(s, w0); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); - unsigned short r = (unsigned short)~_mm_movemask_epi8(x); - if (r != 0) { // some of characters may be non-whitespace -#ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; -#else - return p + __builtin_ffs(r) - 1; -#endif - } - } -} - -#endif // RAPIDJSON_SSE2 - -#ifdef RAPIDJSON_SIMD -//! Template function specialization for InsituStringStream -template<> inline void SkipWhitespace(InsituStringStream& is) { - is.src_ = const_cast(SkipWhitespace_SIMD(is.src_)); -} - -//! Template function specialization for StringStream -template<> inline void SkipWhitespace(StringStream& is) { - is.src_ = SkipWhitespace_SIMD(is.src_); -} -#endif // RAPIDJSON_SIMD - -/////////////////////////////////////////////////////////////////////////////// -// GenericReader - -//! SAX-style JSON parser. Use \ref Reader for UTF8 encoding and default allocator. -/*! GenericReader parses JSON text from a stream, and send events synchronously to an - object implementing Handler concept. - - It needs to allocate a stack for storing a single decoded string during - non-destructive parsing. - - For in-situ parsing, the decoded string is directly written to the source - text string, no temporary buffer is required. - - A GenericReader object can be reused for parsing multiple JSON text. - - \tparam SourceEncoding Encoding of the input stream. - \tparam TargetEncoding Encoding of the parse output. - \tparam StackAllocator Allocator type for stack. -*/ -template -class GenericReader { -public: - typedef typename SourceEncoding::Ch Ch; //!< SourceEncoding character type - - //! Constructor. - /*! \param allocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing) - \param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing) - */ - GenericReader(StackAllocator* stackAllocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(stackAllocator, stackCapacity), parseResult_() {} - - //! Parse JSON text. - /*! \tparam parseFlags Combination of \ref ParseFlag. - \tparam InputStream Type of input stream, implementing Stream concept. - \tparam Handler Type of handler, implementing Handler concept. - \param is Input stream to be parsed. - \param handler The handler to receive events. - \return Whether the parsing is successful. - */ - template - ParseResult Parse(InputStream& is, Handler& handler) { - if (parseFlags & kParseIterativeFlag) - return IterativeParse(is, handler); - - parseResult_.Clear(); - - ClearStackOnExit scope(*this); - - SkipWhitespace(is); - - if (is.Peek() == '\0') { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell()); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - } - else { - ParseValue(is, handler); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - - if (!(parseFlags & kParseStopWhenDoneFlag)) { - SkipWhitespace(is); - - if (is.Peek() != '\0') { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - } - } - } - - return parseResult_; - } - - //! Parse JSON text (with \ref kParseDefaultFlags) - /*! \tparam InputStream Type of input stream, implementing Stream concept - \tparam Handler Type of handler, implementing Handler concept. - \param is Input stream to be parsed. - \param handler The handler to receive events. - \return Whether the parsing is successful. - */ - template - ParseResult Parse(InputStream& is, Handler& handler) { - return Parse(is, handler); - } - - //! Whether a parse error has occured in the last parsing. - bool HasParseError() const { return parseResult_.IsError(); } - - //! Get the \ref ParseErrorCode of last parsing. - ParseErrorCode GetParseErrorCode() const { return parseResult_.Code(); } - - //! Get the position of last parsing error in input, 0 otherwise. - size_t GetErrorOffset() const { return parseResult_.Offset(); } - -protected: - void SetParseError(ParseErrorCode code, size_t offset) { parseResult_.Set(code, offset); } - -private: - // Prohibit copy constructor & assignment operator. - GenericReader(const GenericReader&); - GenericReader& operator=(const GenericReader&); - - void ClearStack() { stack_.Clear(); } - - // clear stack on any exit from ParseStream, e.g. due to exception - struct ClearStackOnExit { - explicit ClearStackOnExit(GenericReader& r) : r_(r) {} - ~ClearStackOnExit() { r_.ClearStack(); } - private: - GenericReader& r_; - ClearStackOnExit(const ClearStackOnExit&); - ClearStackOnExit& operator=(const ClearStackOnExit&); - }; - - // Parse object: { string : value, ... } - template - void ParseObject(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == '{'); - is.Take(); // Skip '{' - - if (!handler.StartObject()) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - - SkipWhitespace(is); - - if (is.Peek() == '}') { - is.Take(); - if (!handler.EndObject(0)) // empty object - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - return; - } - - for (SizeType memberCount = 0;;) { - if (is.Peek() != '"') - RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); - - ParseString(is, handler, true); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - - SkipWhitespace(is); - - if (is.Take() != ':') - RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); - - SkipWhitespace(is); - - ParseValue(is, handler); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - - SkipWhitespace(is); - - ++memberCount; - - switch (is.Take()) { - case ',': SkipWhitespace(is); break; - case '}': - if (!handler.EndObject(memberCount)) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - else - return; - default: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); - } - } - } - - // Parse array: [ value, ... ] - template - void ParseArray(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == '['); - is.Take(); // Skip '[' - - if (!handler.StartArray()) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - - SkipWhitespace(is); - - if (is.Peek() == ']') { - is.Take(); - if (!handler.EndArray(0)) // empty array - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - return; - } - - for (SizeType elementCount = 0;;) { - ParseValue(is, handler); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - - ++elementCount; - SkipWhitespace(is); - - switch (is.Take()) { - case ',': SkipWhitespace(is); break; - case ']': - if (!handler.EndArray(elementCount)) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - else - return; - default: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); - } - } - } - - template - void ParseNull(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == 'n'); - is.Take(); - - if (is.Take() == 'u' && is.Take() == 'l' && is.Take() == 'l') { - if (!handler.Null()) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); - } - - template - void ParseTrue(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == 't'); - is.Take(); - - if (is.Take() == 'r' && is.Take() == 'u' && is.Take() == 'e') { - if (!handler.Bool(true)) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); - } - - template - void ParseFalse(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == 'f'); - is.Take(); - - if (is.Take() == 'a' && is.Take() == 'l' && is.Take() == 's' && is.Take() == 'e') { - if (!handler.Bool(false)) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); - } - - // Helper function to parse four hexidecimal digits in \uXXXX in ParseString(). - template - unsigned ParseHex4(InputStream& is) { - unsigned codepoint = 0; - for (int i = 0; i < 4; i++) { - Ch c = is.Take(); - codepoint <<= 4; - codepoint += static_cast(c); - if (c >= '0' && c <= '9') - codepoint -= '0'; - else if (c >= 'A' && c <= 'F') - codepoint -= 'A' - 10; - else if (c >= 'a' && c <= 'f') - codepoint -= 'a' - 10; - else { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, is.Tell() - 1); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(0); - } - } - return codepoint; - } - - class StackStream { - public: - typedef typename TargetEncoding::Ch Ch; - - StackStream(internal::Stack& stack) : stack_(stack), length_(0) {} - RAPIDJSON_FORCEINLINE void Put(Ch c) { - *stack_.template Push() = c; - ++length_; - } - internal::Stack& stack_; - SizeType length_; - - private: - StackStream(const StackStream&); - StackStream& operator=(const StackStream&); - }; - - // Parse string and generate String event. Different code paths for kParseInsituFlag. - template - void ParseString(InputStream& is, Handler& handler, bool isKey = false) { - internal::StreamLocalCopy copy(is); - InputStream& s(copy.s); - - bool success = false; - if (parseFlags & kParseInsituFlag) { - typename InputStream::Ch *head = s.PutBegin(); - ParseStringToStream(s, s); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - size_t length = s.PutEnd(head) - 1; - RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); - const typename TargetEncoding::Ch* const str = (typename TargetEncoding::Ch*)head; - success = (isKey ? handler.Key(str, SizeType(length), false) : handler.String(str, SizeType(length), false)); - } - else { - StackStream stackStream(stack_); - ParseStringToStream(s, stackStream); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - const typename TargetEncoding::Ch* const str = stack_.template Pop(stackStream.length_); - success = (isKey ? handler.Key(str, stackStream.length_ - 1, true) : handler.String(str, stackStream.length_ - 1, true)); - } - if (!success) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); - } - - // Parse string to an output is - // This function handles the prefix/suffix double quotes, escaping, and optional encoding validation. - template - RAPIDJSON_FORCEINLINE void ParseStringToStream(InputStream& is, OutputStream& os) { -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - static const char escape[256] = { - Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', - Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, - 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, - 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 - }; -#undef Z16 -//!@endcond - - RAPIDJSON_ASSERT(is.Peek() == '\"'); - is.Take(); // Skip '\"' - - for (;;) { - Ch c = is.Peek(); - if (c == '\\') { // Escape - is.Take(); - Ch e = is.Take(); - if ((sizeof(Ch) == 1 || unsigned(e) < 256) && escape[(unsigned char)e]) { - os.Put(escape[(unsigned char)e]); - } - else if (e == 'u') { // Unicode - unsigned codepoint = ParseHex4(is); - if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { - // Handle UTF-16 surrogate pair - if (is.Take() != '\\' || is.Take() != 'u') - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, is.Tell() - 2); - unsigned codepoint2 = ParseHex4(is); - if (codepoint2 < 0xDC00 || codepoint2 > 0xDFFF) - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, is.Tell() - 2); - codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; - } - TEncoding::Encode(os, codepoint); - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell() - 1); - } - else if (c == '"') { // Closing double quote - is.Take(); - os.Put('\0'); // null-terminate the string - return; - } - else if (c == '\0') - RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell() - 1); - else if ((unsigned)c < 0x20) // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF - RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell() - 1); - else { - if (parseFlags & kParseValidateEncodingFlag ? - !Transcoder::Validate(is, os) : - !Transcoder::Transcode(is, os)) - RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, is.Tell()); - } - } - } - - inline double StrtodFastPath(double significand, int exp) { - // Fast path only works on limited range of values. - // But for simplicity and performance, currently only implement this. - // see http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - if (exp < -308) - return 0.0; - else if (exp >= 0) - return significand * internal::Pow10(exp); - else - return significand / internal::Pow10(-exp); - } - - template - void ParseNumber(InputStream& is, Handler& handler) { - internal::StreamLocalCopy copy(is); - InputStream& s(copy.s); - - // Parse minus - bool minus = false; - if (s.Peek() == '-') { - minus = true; - s.Take(); - } - - // Parse int: zero / ( digit1-9 *DIGIT ) - unsigned i = 0; - uint64_t i64 = 0; - bool use64bit = false; - if (s.Peek() == '0') { - i = 0; - s.Take(); - } - else if (s.Peek() >= '1' && s.Peek() <= '9') { - i = static_cast(s.Take() - '0'); - - if (minus) - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i >= 214748364) { // 2^31 = 2147483648 - if (i != 214748364 || s.Peek() > '8') { - i64 = i; - use64bit = true; - break; - } - } - i = i * 10 + static_cast(s.Take() - '0'); - } - else - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i >= 429496729) { // 2^32 - 1 = 4294967295 - if (i != 429496729 || s.Peek() > '5') { - i64 = i; - use64bit = true; - break; - } - } - i = i * 10 + static_cast(s.Take() - '0'); - } - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); - - // Parse 64bit int - double d = 0.0; - bool useDouble = false; - if (use64bit) { - if (minus) - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC)) // 2^63 = 9223372036854775808 - if (i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8') { - d = (double)i64; - useDouble = true; - break; - } - i64 = i64 * 10 + static_cast(s.Take() - '0'); - } - else - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999)) // 2^64 - 1 = 18446744073709551615 - if (i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5') { - d = (double)i64; - useDouble = true; - break; - } - i64 = i64 * 10 + static_cast(s.Take() - '0'); - } - } - - // Force double for big integer - if (useDouble) { - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (d >= 1.7976931348623157e307) // DBL_MAX / 10.0 - RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, s.Tell()); - d = d * 10 + (s.Take() - '0'); - } - } - - // Parse frac = decimal-point 1*DIGIT - int expFrac = 0; - if (s.Peek() == '.') { - s.Take(); - -#if RAPIDJSON_64BIT - // Use i64 to store significand in 64-bit architecture - if (!useDouble) { - if (!use64bit) - i64 = i; - - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999)) - break; - else { - i64 = i64 * 10 + static_cast(s.Take() - '0'); - --expFrac; - } - } - - d = (double)i64; - } -#else - // Use double to store significand in 32-bit architecture - if (!useDouble) - d = use64bit ? (double)i64 : (double)i; -#endif - useDouble = true; - - while (s.Peek() >= '0' && s.Peek() <= '9') { - d = d * 10 + (s.Take() - '0'); - --expFrac; - } - - if (expFrac == 0) - RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissFraction, s.Tell()); - } - - // Parse exp = e [ minus / plus ] 1*DIGIT - int exp = 0; - if (s.Peek() == 'e' || s.Peek() == 'E') { - if (!useDouble) { - d = use64bit ? (double)i64 : (double)i; - useDouble = true; - } - s.Take(); - - bool expMinus = false; - if (s.Peek() == '+') - s.Take(); - else if (s.Peek() == '-') { - s.Take(); - expMinus = true; - } - - if (s.Peek() >= '0' && s.Peek() <= '9') { - exp = s.Take() - '0'; - while (s.Peek() >= '0' && s.Peek() <= '9') { - exp = exp * 10 + (s.Take() - '0'); - if (exp > 308 && !expMinus) // exp > 308 should be rare, so it should be checked first. - RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, s.Tell()); - } - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissExponent, s.Tell()); - - if (expMinus) - exp = -exp; - } - - // Finish parsing, call event according to the type of number. - bool cont = true; - if (useDouble) { - int expSum = exp + expFrac; - if (expSum < -308) { - // Prevent expSum < -308, making Pow10(expSum) = 0 - d = StrtodFastPath(d, exp); - d = StrtodFastPath(d, expFrac); - } - else - d = StrtodFastPath(d, expSum); - - cont = handler.Double(minus ? -d : d); - } - else { - if (use64bit) { - if (minus) - cont = handler.Int64(-(int64_t)i64); - else - cont = handler.Uint64(i64); - } - else { - if (minus) - cont = handler.Int(-(int)i); - else - cont = handler.Uint(i); - } - } - if (!cont) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); - } - - // Parse any JSON value - template - void ParseValue(InputStream& is, Handler& handler) { - switch (is.Peek()) { - case 'n': ParseNull (is, handler); break; - case 't': ParseTrue (is, handler); break; - case 'f': ParseFalse (is, handler); break; - case '"': ParseString(is, handler); break; - case '{': ParseObject(is, handler); break; - case '[': ParseArray (is, handler); break; - default : ParseNumber(is, handler); - } - } - - // Iterative Parsing - - // States - enum IterativeParsingState { - IterativeParsingStartState = 0, - IterativeParsingFinishState, - IterativeParsingErrorState, - - // Object states - IterativeParsingObjectInitialState, - IterativeParsingMemberKeyState, - IterativeParsingKeyValueDelimiterState, - IterativeParsingMemberValueState, - IterativeParsingMemberDelimiterState, - IterativeParsingObjectFinishState, - - // Array states - IterativeParsingArrayInitialState, - IterativeParsingElementState, - IterativeParsingElementDelimiterState, - IterativeParsingArrayFinishState, - - // Single value state - IterativeParsingValueState, - - cIterativeParsingStateCount - }; - - // Tokens - enum Token { - LeftBracketToken = 0, - RightBracketToken, - - LeftCurlyBracketToken, - RightCurlyBracketToken, - - CommaToken, - ColonToken, - - StringToken, - FalseToken, - TrueToken, - NullToken, - NumberToken, - - kTokenCount - }; - - RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) { - -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#define N NumberToken -#define N16 N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N - // Maps from ASCII to Token - static const unsigned char tokenMap[256] = { - N16, // 00~0F - N16, // 10~1F - N, N, StringToken, N, N, N, N, N, N, N, N, N, CommaToken, N, N, N, // 20~2F - N, N, N, N, N, N, N, N, N, N, ColonToken, N, N, N, N, N, // 30~3F - N16, // 40~4F - N, N, N, N, N, N, N, N, N, N, N, LeftBracketToken, N, RightBracketToken, N, N, // 50~5F - N, N, N, N, N, N, FalseToken, N, N, N, N, N, N, N, NullToken, N, // 60~6F - N, N, N, N, TrueToken, N, N, N, N, N, N, LeftCurlyBracketToken, N, RightCurlyBracketToken, N, N, // 70~7F - N16, N16, N16, N16, N16, N16, N16, N16 // 80~FF - }; -#undef N -#undef N16 -//!@endcond - - if (sizeof(Ch) == 1 || unsigned(c) < 256) - return (Token)tokenMap[(unsigned char)c]; - else - return NumberToken; - } - - RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) { - // current state x one lookahead token -> new state - static const char G[cIterativeParsingStateCount][kTokenCount] = { - // Start - { - IterativeParsingArrayInitialState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingValueState, // String - IterativeParsingValueState, // False - IterativeParsingValueState, // True - IterativeParsingValueState, // Null - IterativeParsingValueState // Number - }, - // Finish(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // Error(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // ObjectInitial - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingObjectFinishState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberKeyState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // MemberKey - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingKeyValueDelimiterState, // Colon - IterativeParsingErrorState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // KeyValueDelimiter - { - IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) - IterativeParsingErrorState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberValueState, // String - IterativeParsingMemberValueState, // False - IterativeParsingMemberValueState, // True - IterativeParsingMemberValueState, // Null - IterativeParsingMemberValueState // Number - }, - // MemberValue - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingObjectFinishState, // Right curly bracket - IterativeParsingMemberDelimiterState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingErrorState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // MemberDelimiter - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberKeyState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // ObjectFinish(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // ArrayInitial - { - IterativeParsingArrayInitialState, // Left bracket(push Element state) - IterativeParsingArrayFinishState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push Element state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingElementState, // String - IterativeParsingElementState, // False - IterativeParsingElementState, // True - IterativeParsingElementState, // Null - IterativeParsingElementState // Number - }, - // Element - { - IterativeParsingErrorState, // Left bracket - IterativeParsingArrayFinishState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingErrorState, // Right curly bracket - IterativeParsingElementDelimiterState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingErrorState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // ElementDelimiter - { - IterativeParsingArrayInitialState, // Left bracket(push Element state) - IterativeParsingErrorState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push Element state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingElementState, // String - IterativeParsingElementState, // False - IterativeParsingElementState, // True - IterativeParsingElementState, // Null - IterativeParsingElementState // Number - }, - // ArrayFinish(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // Single Value (sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - } - }; // End of G - - return (IterativeParsingState)G[state][token]; - } - - // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). - // May return a new state on state pop. - template - RAPIDJSON_FORCEINLINE IterativeParsingState Transit(IterativeParsingState src, Token token, IterativeParsingState dst, InputStream& is, Handler& handler) { - switch (dst) { - case IterativeParsingStartState: - RAPIDJSON_ASSERT(false); - return IterativeParsingErrorState; - - case IterativeParsingFinishState: - return dst; - - case IterativeParsingErrorState: - return dst; - - case IterativeParsingObjectInitialState: - case IterativeParsingArrayInitialState: - { - // Push the state(Element or MemeberValue) if we are nested in another array or value of member. - // In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop. - IterativeParsingState n = src; - if (src == IterativeParsingArrayInitialState || src == IterativeParsingElementDelimiterState) - n = IterativeParsingElementState; - else if (src == IterativeParsingKeyValueDelimiterState) - n = IterativeParsingMemberValueState; - // Push current state. - *stack_.template Push(1) = n; - // Initialize and push the member/element count. - *stack_.template Push(1) = 0; - // Call handler - bool hr = (dst == IterativeParsingObjectInitialState) ? handler.StartObject() : handler.StartArray(); - // On handler short circuits the parsing. - if (!hr) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); - return IterativeParsingErrorState; - } - else { - is.Take(); - return dst; - } - } - - case IterativeParsingMemberKeyState: - ParseString(is, handler, true); - if (HasParseError()) - return IterativeParsingErrorState; - else - return dst; - - case IterativeParsingKeyValueDelimiterState: - if (token == ColonToken) { - is.Take(); - return dst; - } - else - return IterativeParsingErrorState; - - case IterativeParsingMemberValueState: - // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. - ParseValue(is, handler); - if (HasParseError()) { - return IterativeParsingErrorState; - } - return dst; - - case IterativeParsingElementState: - // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. - ParseValue(is, handler); - if (HasParseError()) { - return IterativeParsingErrorState; - } - return dst; - - case IterativeParsingMemberDelimiterState: - case IterativeParsingElementDelimiterState: - is.Take(); - // Update member/element count. - *stack_.template Top() = *stack_.template Top() + 1; - return dst; - - case IterativeParsingObjectFinishState: - { - // Get member count. - SizeType c = *stack_.template Pop(1); - // If the object is not empty, count the last member. - if (src == IterativeParsingMemberValueState) - ++c; - // Restore the state. - IterativeParsingState n = static_cast(*stack_.template Pop(1)); - // Transit to Finish state if this is the topmost scope. - if (n == IterativeParsingStartState) - n = IterativeParsingFinishState; - // Call handler - bool hr = handler.EndObject(c); - // On handler short circuits the parsing. - if (!hr) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); - return IterativeParsingErrorState; - } - else { - is.Take(); - return n; - } - } - - case IterativeParsingArrayFinishState: - { - // Get element count. - SizeType c = *stack_.template Pop(1); - // If the array is not empty, count the last element. - if (src == IterativeParsingElementState) - ++c; - // Restore the state. - IterativeParsingState n = static_cast(*stack_.template Pop(1)); - // Transit to Finish state if this is the topmost scope. - if (n == IterativeParsingStartState) - n = IterativeParsingFinishState; - // Call handler - bool hr = handler.EndArray(c); - // On handler short circuits the parsing. - if (!hr) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); - return IterativeParsingErrorState; - } - else { - is.Take(); - return n; - } - } - - case IterativeParsingValueState: - // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. - ParseValue(is, handler); - if (HasParseError()) { - return IterativeParsingErrorState; - } - return IterativeParsingFinishState; - - default: - RAPIDJSON_ASSERT(false); - return IterativeParsingErrorState; - } - } - - template - void HandleError(IterativeParsingState src, InputStream& is) { - if (HasParseError()) { - // Error flag has been set. - return; - } - - switch (src) { - case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); - case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); - case IterativeParsingObjectInitialState: - case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); - case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); - case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); - case IterativeParsingElementState: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); - default: RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); - } - } - - template - ParseResult IterativeParse(InputStream& is, Handler& handler) { - parseResult_.Clear(); - ClearStackOnExit scope(*this); - IterativeParsingState state = IterativeParsingStartState; - - SkipWhitespace(is); - while (is.Peek() != '\0') { - Token t = Tokenize(is.Peek()); - IterativeParsingState n = Predict(state, t); - IterativeParsingState d = Transit(state, t, n, is, handler); - - if (d == IterativeParsingErrorState) { - HandleError(state, is); - break; - } - - state = d; - - // Do not further consume streams if a root JSON has been parsed. - if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) - break; - - SkipWhitespace(is); - } - - // Handle the end of file. - if (state != IterativeParsingFinishState) - HandleError(state, is); - - return parseResult_; - } - - static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. - internal::Stack stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. - ParseResult parseResult_; -}; // class GenericReader - -//! Reader with UTF8 encoding and default allocator. -typedef GenericReader, UTF8<> > Reader; - -} // namespace rapidjson - -#ifdef _MSC_VER -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_READER_H_ diff --git a/tools/optimizer/rapidjson/stringbuffer.h b/tools/optimizer/rapidjson/stringbuffer.h deleted file mode 100644 index 55124f117adb9..0000000000000 --- a/tools/optimizer/rapidjson/stringbuffer.h +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_STRINGBUFFER_H_ -#define RAPIDJSON_STRINGBUFFER_H_ - -#include "rapidjson.h" -#include "internal/stack.h" - -namespace rapidjson { - -//! Represents an in-memory output stream. -/*! - \tparam Encoding Encoding of the stream. - \tparam Allocator type for allocating memory buffer. - \note implements Stream concept -*/ -template -struct GenericStringBuffer { - typedef typename Encoding::Ch Ch; - - GenericStringBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} - - void Put(Ch c) { *stack_.template Push() = c; } - void Flush() {} - - void Clear() { stack_.Clear(); } - void ShrinkToFit() { - // Push and pop a null terminator. This is safe. - *stack_.template Push() = '\0'; - stack_.ShrinkToFit(); - stack_.template Pop(1); - } - Ch* Push(size_t count) { return stack_.template Push(count); } - void Pop(size_t count) { stack_.template Pop(count); } - - const Ch* GetString() const { - // Push and pop a null terminator. This is safe. - *stack_.template Push() = '\0'; - stack_.template Pop(1); - - return stack_.template Bottom(); - } - - size_t GetSize() const { return stack_.GetSize(); } - - static const size_t kDefaultCapacity = 256; - mutable internal::Stack stack_; -}; - -//! String buffer with UTF8 encoding -typedef GenericStringBuffer > StringBuffer; - -//! Implement specialized version of PutN() with memset() for better performance. -template<> -inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { - std::memset(stream.stack_.Push(n), c, n * sizeof(c)); -} - -} // namespace rapidjson - -#endif // RAPIDJSON_STRINGBUFFER_H_ diff --git a/tools/optimizer/rapidjson/writer.h b/tools/optimizer/rapidjson/writer.h deleted file mode 100644 index fb6601ef97684..0000000000000 --- a/tools/optimizer/rapidjson/writer.h +++ /dev/null @@ -1,389 +0,0 @@ -// Copyright (C) 2011 Milo Yip -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef RAPIDJSON_WRITER_H_ -#define RAPIDJSON_WRITER_H_ - -#include "rapidjson.h" -#include "internal/stack.h" -#include "internal/strfunc.h" -#include "internal/dtoa.h" -#include "internal/itoa.h" -#include "stringbuffer.h" -#include // placement new - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant -#endif - -namespace rapidjson { - -//! JSON writer -/*! Writer implements the concept Handler. - It generates JSON text by events to an output os. - - User may programmatically calls the functions of a writer to generate JSON text. - - On the other side, a writer can also be passed to objects that generates events, - - for example Reader::Parse() and Document::Accept(). - - \tparam OutputStream Type of output stream. - \tparam SourceEncoding Encoding of source string. - \tparam TargetEncoding Encoding of output stream. - \tparam StackAllocator Type of allocator for allocating memory of stack. - \note implements Handler concept -*/ -template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator> -class Writer { -public: - typedef typename SourceEncoding::Ch Ch; - - //! Constructor - /*! \param os Output stream. - \param allocator User supplied allocator. If it is null, it will create a private one. - \param levelDepth Initial capacity of stack. - */ - Writer(OutputStream& os, StackAllocator* stackAllocator = 0, size_t levelDepth = kDefaultLevelDepth) : - os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), hasRoot_(false) {} - - Writer(StackAllocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : - os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), hasRoot_(false) {} - - //! Reset the writer with a new stream. - /*! - This function reset the writer with a new stream and default settings, - in order to make a Writer object reusable for output multiple JSONs. - - \param os New output stream. - \code - Writer writer(os1); - writer.StartObject(); - // ... - writer.EndObject(); - - writer.Reset(os2); - writer.StartObject(); - // ... - writer.EndObject(); - \endcode - */ - void Reset(OutputStream& os) { - os_ = &os; - hasRoot_ = false; - level_stack_.Clear(); - } - - //! Checks whether the output is a complete JSON. - /*! - A complete JSON has a complete root object or array. - */ - bool IsComplete() const { - return hasRoot_ && level_stack_.Empty(); - } - - /*!@name Implementation of Handler - \see Handler - */ - //@{ - - bool Null() { Prefix(kNullType); return WriteNull(); } - bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return WriteBool(b); } - bool Int(int i) { Prefix(kNumberType); return WriteInt(i); } - bool Uint(unsigned u) { Prefix(kNumberType); return WriteUint(u); } - bool Int64(int64_t i64) { Prefix(kNumberType); return WriteInt64(i64); } - bool Uint64(uint64_t u64) { Prefix(kNumberType); return WriteUint64(u64); } - - //! Writes the given \c double value to the stream - /*! - \param d The value to be written. - \return Whether it is succeed. - */ - bool Double(double d) { Prefix(kNumberType); return WriteDouble(d); } - - bool String(const Ch* str, SizeType length, bool copy = false) { - (void)copy; - Prefix(kStringType); - return WriteString(str, length); - } - - bool StartObject() { - Prefix(kObjectType); - new (level_stack_.template Push()) Level(false); - return WriteStartObject(); - } - - bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } - - bool EndObject(SizeType memberCount = 0) { - (void)memberCount; - RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); - RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); - level_stack_.template Pop(1); - bool ret = WriteEndObject(); - if (level_stack_.Empty()) // end of json text - os_->Flush(); - return ret; - } - - bool StartArray() { - Prefix(kArrayType); - new (level_stack_.template Push()) Level(true); - return WriteStartArray(); - } - - bool EndArray(SizeType elementCount = 0) { - (void)elementCount; - RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); - RAPIDJSON_ASSERT(level_stack_.template Top()->inArray); - level_stack_.template Pop(1); - bool ret = WriteEndArray(); - if (level_stack_.Empty()) // end of json text - os_->Flush(); - return ret; - } - //@} - - /*! @name Convenience extensions */ - //@{ - - //! Simpler but slower overload. - bool String(const Ch* str) { return String(str, internal::StrLen(str)); } - bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } - - //@} - -protected: - //! Information for each nested level - struct Level { - Level(bool inArray_) : valueCount(0), inArray(inArray_) {} - size_t valueCount; //!< number of values in this level - bool inArray; //!< true if in array, otherwise in object - }; - - static const size_t kDefaultLevelDepth = 32; - - bool WriteNull() { - os_->Put('n'); os_->Put('u'); os_->Put('l'); os_->Put('l'); return true; - } - - bool WriteBool(bool b) { - if (b) { - os_->Put('t'); os_->Put('r'); os_->Put('u'); os_->Put('e'); - } - else { - os_->Put('f'); os_->Put('a'); os_->Put('l'); os_->Put('s'); os_->Put('e'); - } - return true; - } - - bool WriteInt(int i) { - char buffer[11]; - const char* end = internal::i32toa(i, buffer); - for (const char* p = buffer; p != end; ++p) - os_->Put(*p); - return true; - } - - bool WriteUint(unsigned u) { - char buffer[10]; - const char* end = internal::u32toa(u, buffer); - for (const char* p = buffer; p != end; ++p) - os_->Put(*p); - return true; - } - - bool WriteInt64(int64_t i64) { - char buffer[21]; - const char* end = internal::i64toa(i64, buffer); - for (const char* p = buffer; p != end; ++p) - os_->Put(*p); - return true; - } - - bool WriteUint64(uint64_t u64) { - char buffer[20]; - char* end = internal::u64toa(u64, buffer); - for (char* p = buffer; p != end; ++p) - os_->Put(*p); - return true; - } - - bool WriteDouble(double d) { - char buffer[25]; - char* end = internal::dtoa(d, buffer); - for (char* p = buffer; p != end; ++p) - os_->Put(*p); - return true; - } - - bool WriteString(const Ch* str, SizeType length) { - static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; - static const char escape[256] = { -#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - //0 1 2 3 4 5 6 7 8 9 A B C D E F - 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'b', 't', 'n', 'u', 'f', 'r', 'u', 'u', // 00 - 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', // 10 - 0, 0, '"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20 - Z16, Z16, // 30~4F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, // 50 - Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 // 60~FF -#undef Z16 - }; - - os_->Put('\"'); - GenericStringStream is(str); - while (is.Tell() < length) { - const Ch c = is.Peek(); - if (!TargetEncoding::supportUnicode && (unsigned)c >= 0x80) { - // Unicode escaping - unsigned codepoint; - if (!SourceEncoding::Decode(is, &codepoint)) - return false; - os_->Put('\\'); - os_->Put('u'); - if (codepoint <= 0xD7FF || (codepoint >= 0xE000 && codepoint <= 0xFFFF)) { - os_->Put(hexDigits[(codepoint >> 12) & 15]); - os_->Put(hexDigits[(codepoint >> 8) & 15]); - os_->Put(hexDigits[(codepoint >> 4) & 15]); - os_->Put(hexDigits[(codepoint ) & 15]); - } - else if (codepoint >= 0x010000 && codepoint <= 0x10FFFF) { - // Surrogate pair - unsigned s = codepoint - 0x010000; - unsigned lead = (s >> 10) + 0xD800; - unsigned trail = (s & 0x3FF) + 0xDC00; - os_->Put(hexDigits[(lead >> 12) & 15]); - os_->Put(hexDigits[(lead >> 8) & 15]); - os_->Put(hexDigits[(lead >> 4) & 15]); - os_->Put(hexDigits[(lead ) & 15]); - os_->Put('\\'); - os_->Put('u'); - os_->Put(hexDigits[(trail >> 12) & 15]); - os_->Put(hexDigits[(trail >> 8) & 15]); - os_->Put(hexDigits[(trail >> 4) & 15]); - os_->Put(hexDigits[(trail ) & 15]); - } - else - return false; // invalid code point - } - else if ((sizeof(Ch) == 1 || (unsigned)c < 256) && escape[(unsigned char)c]) { - is.Take(); - os_->Put('\\'); - os_->Put(escape[(unsigned char)c]); - if (escape[(unsigned char)c] == 'u') { - os_->Put('0'); - os_->Put('0'); - os_->Put(hexDigits[(unsigned char)c >> 4]); - os_->Put(hexDigits[(unsigned char)c & 0xF]); - } - } - else - Transcoder::Transcode(is, *os_); - } - os_->Put('\"'); - return true; - } - - bool WriteStartObject() { os_->Put('{'); return true; } - bool WriteEndObject() { os_->Put('}'); return true; } - bool WriteStartArray() { os_->Put('['); return true; } - bool WriteEndArray() { os_->Put(']'); return true; } - - void Prefix(Type type) { - (void)type; - if (level_stack_.GetSize() != 0) { // this value is not at root - Level* level = level_stack_.template Top(); - if (level->valueCount > 0) { - if (level->inArray) - os_->Put(','); // add comma if it is not the first element in array - else // in object - os_->Put((level->valueCount % 2 == 0) ? ',' : ':'); - } - if (!level->inArray && level->valueCount % 2 == 0) - RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name - level->valueCount++; - } - else { - RAPIDJSON_ASSERT(!hasRoot_); // Should only has one and only one root. - hasRoot_ = true; - } - } - - OutputStream* os_; - internal::Stack level_stack_; - bool hasRoot_; - -private: - // Prohibit copy constructor & assignment operator. - Writer(const Writer&); - Writer& operator=(const Writer&); -}; - -// Full specialization for StringStream to prevent memory copying - -template<> -inline bool Writer::WriteInt(int i) { - char *buffer = os_->Push(11); - const char* end = internal::i32toa(i, buffer); - os_->Pop(11 - (end - buffer)); - return true; -} - -template<> -inline bool Writer::WriteUint(unsigned u) { - char *buffer = os_->Push(10); - const char* end = internal::u32toa(u, buffer); - os_->Pop(10 - (end - buffer)); - return true; -} - -template<> -inline bool Writer::WriteInt64(int64_t i64) { - char *buffer = os_->Push(21); - const char* end = internal::i64toa(i64, buffer); - os_->Pop(21 - (end - buffer)); - return true; -} - -template<> -inline bool Writer::WriteUint64(uint64_t u) { - char *buffer = os_->Push(20); - const char* end = internal::u64toa(u, buffer); - os_->Pop(20 - (end - buffer)); - return true; -} - -template<> -inline bool Writer::WriteDouble(double d) { - char *buffer = os_->Push(25); - char* end = internal::dtoa(d, buffer); - os_->Pop(25 - (end - buffer)); - return true; -} - -} // namespace rapidjson - -#ifdef _MSC_VER -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_RAPIDJSON_H_ From 26271c5318650dbd0ed2fd2de75e7ac8fab341ac Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sat, 8 Nov 2014 11:03:00 -0800 Subject: [PATCH 070/161] derecurse traversePre|Post --- tools/optimizer/optimizer.cpp | 50 ++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index e8a7694a74093..257128e62fac6 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -44,37 +44,63 @@ std::string getIntStr(int x) { // Traverses the children of a node. // visit() receives a reference, so it can modify the value being visited directly. -// N.B. we allow replacing arrays in place, and consider strings immutable! -// TODO: stoppable version of this void traverseChildren(Ref node, std::function visit) { - //dump("TC", node); if (!node->isArray()) return; int size = node->size(); - //printf("size: %d\n", size); for (int i = 0; i < size; i++) { - //printf(" %d:\n", i); Ref subnode = node[i]; if (subnode->isArray() and subnode->size() > 0) { - //printf(" go\n"); visit(subnode); } } } +struct TraverseInfo { + Ref node; + int index; +}; + +#define visitable(node) (node->isArray() and node->size() > 0) + // Traverse, calling visit before the children void traversePre(Ref node, std::function visit) { visit(node); - traverseChildren(node, [&visit](Ref node) { - traversePre(node, visit); - }); + std::vector stack; + stack.push_back({ node, 0 }); + while (stack.size() > 0) { + TraverseInfo& top = stack.back(); + if (top.index < top.node->size()) { + Ref sub = top.node[top.index]; + top.index++; + if (visitable(sub)) { + visit(sub); + stack.push_back({ sub, 0 }); + } + } else { + stack.pop_back(); + } + } } // Traverse, calling visitPre before the children and visitPost after void traversePrePost(Ref node, std::function visitPre, std::function visitPost) { visitPre(node); - traverseChildren(node, [&visitPre, &visitPost](Ref node) { - traversePrePost(node, visitPre, visitPost); - }); + std::vector stack; + stack.push_back({ node, 0 }); + while (stack.size() > 0) { + TraverseInfo& top = stack.back(); + if (top.index < top.node->size()) { + Ref sub = top.node[top.index]; + top.index++; + if (visitable(sub)) { + visitPre(sub); + stack.push_back({ sub, 0 }); + } + } else { + visitPost(top.node); + stack.pop_back(); + } + } visitPost(node); } From 0032c426759af092a5efa5d47585ca6c631ac863 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sat, 8 Nov 2014 12:38:48 -0800 Subject: [PATCH 071/161] remove traverseChildren, which forces recursion --- tools/optimizer/optimizer.cpp | 65 ++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 257128e62fac6..0d018a459eec1 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -42,19 +42,6 @@ std::string getIntStr(int x) { // Traversals -// Traverses the children of a node. -// visit() receives a reference, so it can modify the value being visited directly. -void traverseChildren(Ref node, std::function visit) { - if (!node->isArray()) return; - int size = node->size(); - for (int i = 0; i < size; i++) { - Ref subnode = node[i]; - if (subnode->isArray() and subnode->size() > 0) { - visit(subnode); - } - } -} - struct TraverseInfo { Ref node; int index; @@ -104,6 +91,29 @@ void traversePrePost(Ref node, std::function visitPre, std::function visitPost(node); } +// Traverse, calling visitPre before the children and visitPost after. If pre returns false, do not traverse children +void traversePrePostConditional(Ref node, std::function visitPre, std::function visitPost) { + if (!visitPre(node)) return; + std::vector stack; + stack.push_back({ node, 0 }); + while (stack.size() > 0) { + TraverseInfo& top = stack.back(); + if (top.index < top.node->size()) { + Ref sub = top.node[top.index]; + top.index++; + if (visitable(sub)) { + if (visitPre(sub)) { + stack.push_back({ sub, 0 }); + } + } + } else { + visitPost(top.node); + stack.pop_back(); + } + } + visitPost(node); +} + // Traverses all the top-level functions in the document void traverseFunctions(Ref ast, std::function visit) { if (ast[0] == "toplevel") { @@ -831,17 +841,18 @@ void simplifyExpressions(Ref ast) { bool hasTempDoublePtr = false, rerunOrZeroPass = false; - std::function andHeapOpts = [&hasTempDoublePtr, &rerunOrZeroPass, &andHeapOpts](Ref node) { + traversePrePostConditional(ast, [](Ref node) { // Detect trees which should not // be simplified. - Ref type = node[0]; - if (type == "sub" && node[1][0] == "name" && isFunctionTable(node[1][1])) { - return; // do not traverse subchildren here, we should not collapse 55 & 126. + if (node[0] == "sub" && node[1][0] == "name" && isFunctionTable(node[1][1])) { + return false; // do not traverse subchildren here, we should not collapse 55 & 126. } - traverseChildren(node, andHeapOpts); + return true; + }, [&hasTempDoublePtr, &rerunOrZeroPass](Ref node) { // Simplifications are done now so // that we simplify a node's operands before the node itself. This allows // optimizations to cascade. + Ref type = node[0]; if (type == "name") { if (node[1] == "tempDoublePtr") hasTempDoublePtr = true; } else if (type == "binary" && node[1] == "&" && node[3][0] == "num") { @@ -957,8 +968,7 @@ void simplifyExpressions(Ref ast) { } } } - }; - traverseChildren(ast, andHeapOpts); + }); if (rerunOrZeroPass) removeMultipleOrZero(); @@ -1321,12 +1331,12 @@ void simplifyIfs(Ref ast) { void optimizeFrounds(Ref ast) { // collapse fround(fround(..)), which can happen due to elimination // also emit f0 instead of fround(0) (except in returns) - bool inReturn = false; - std::function fix = [&](Ref node) { - bool ret = node[0] == "return"; - if (ret) inReturn = true; - traverseChildren(node, fix); - if (ret) inReturn = false; + bool inReturn = false, currIsReturn = false; + traversePrePost(ast, [&](Ref node) { + currIsReturn = node[0] == "return"; + if (currIsReturn) inReturn = true; + }, [&](Ref node) { + if (currIsReturn) inReturn = false; if (node[0] == "call" && node[1][0] == "name" && node[1][1] == "Math_fround") { Ref arg = node[2][0]; if (arg[0] == "num") { @@ -1335,8 +1345,7 @@ void optimizeFrounds(Ref ast) { safeCopy(node, arg); } } - }; - traverseChildren(ast, fix); + }); } //================== From 99b67c963a8fce4afc7d6b9ceacfcddd19b8f409 Mon Sep 17 00:00:00 2001 From: Aidan Hobson Sayers Date: Fri, 31 Oct 2014 01:32:12 +0000 Subject: [PATCH 072/161] Convert js engine cfs to lists immediately after config read --- emcc | 2 +- tests/test_other.py | 9 ++++----- tests/test_sockets.py | 6 +++--- tools/js_optimizer.py | 1 - tools/shared.py | 22 ++++++++++++++-------- tools/validate_asmjs.py | 2 +- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/emcc b/emcc index 41f91840f73c7..c11dcaa4984c9 100755 --- a/emcc +++ b/emcc @@ -259,7 +259,7 @@ if CONFIGURE_CONFIG or CMAKE_CONFIG: shutil.copyfile(target, target[:-3]) target = target[:-3] src = open(target).read() - full_node = ' '.join(shared.listify(shared.NODE_JS)) + full_node = ' '.join(shared.NODE_JS) if os.path.sep not in full_node: full_node = '/usr/bin/' + full_node # TODO: use whereis etc. And how about non-*NIX? open(target, 'w').write('#!' + full_node + '\n' + src) # add shebang diff --git a/tests/test_other.py b/tests/test_other.py index b444a476f2d1a..7733b5bbed7a9 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -430,7 +430,7 @@ def check_makefile(configuration, dirname): # Run through node, if CMake produced a .js file. if cmake_outputs[i].endswith('.js'): - ret = Popen(listify(NODE_JS) + [tempdirname + '/' + cmake_outputs[i]], stdout=PIPE).communicate()[0] + ret = Popen(NODE_JS + [tempdirname + '/' + cmake_outputs[i]], stdout=PIPE).communicate()[0] self.assertTextDataIdentical(open(cmakelistsdir + '/out.txt', 'r').read().strip(), ret.strip()) finally: os.chdir(path_from_root('tests')) # Move away from the directory we are about to remove. @@ -1238,7 +1238,7 @@ def test_ungetc_fscanf(self): ''') open('my_test.input', 'w').write('abc') Building.emcc('main.cpp', ['--embed-file', 'my_test.input'], output_filename='a.out.js') - self.assertContained('zyx', Popen(listify(JS_ENGINES[0]) + ['a.out.js'], stdout=PIPE, stderr=PIPE).communicate()[0]) + self.assertContained('zyx', Popen(JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).communicate()[0]) def test_abspaths(self): # Includes with absolute paths are generally dangerous, things like -I/usr/.. will get to system local headers, not our portable ones. @@ -1932,7 +1932,7 @@ def test_js_optimizer(self): ['asm', 'ensureLabelSet']), ]: print input - output = Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), input] + passes, stdin=PIPE, stdout=PIPE).communicate()[0] + output = Popen(NODE_JS + [path_from_root('tools', 'js-optimizer.js'), input] + passes, stdin=PIPE, stdout=PIPE).communicate()[0] self.assertIdentical(expected, output.replace('\r\n', '\n').replace('\n\n', '\n')) def test_m_mm(self): @@ -3063,7 +3063,6 @@ def test_returncode(self): ''') Popen([PYTHON, EMCC, 'src.cpp']).communicate() for engine in JS_ENGINES: - engine = listify(engine) print engine process = Popen(engine + ['a.out.js'], stdout=PIPE, stderr=PIPE) output = process.communicate() @@ -4386,6 +4385,6 @@ def test_link_with_a_static(self): def test_require(self): inname = path_from_root('tests', 'hello_world.c') Building.emcc(inname, output_filename='a.out.js') - output = Popen(listify(NODE_JS) + ['-e', 'require("./a.out.js")'], stdout=PIPE, stderr=PIPE).communicate() + output = Popen(NODE_JS + ['-e', 'require("./a.out.js")'], stdout=PIPE, stderr=PIPE).communicate() assert output == ('hello, world!\n \n', ''), 'expected no output, got\n===\nSTDOUT\n%s\n===\nSTDERR\n%s\n===\n' % output diff --git a/tests/test_sockets.py b/tests/test_sockets.py index 90a8202b04e6e..f311ed8910398 100644 --- a/tests/test_sockets.py +++ b/tests/test_sockets.py @@ -80,13 +80,13 @@ def __init__(self, filename, args, listen_port): def __enter__(self): # assuming this is only used for WebSocket tests at the moment, validate that # the ws module is installed - child = Popen(listify(NODE_JS) + ['-e', 'require("ws");']) + child = Popen(NODE_JS + ['-e', 'require("ws");']) child.communicate() assert child.returncode == 0, 'ws module for Node.js not installed. Please run \'npm install\' from %s' % EMSCRIPTEN_ROOT # compile the server Popen([PYTHON, EMCC, path_from_root('tests', self.filename), '-o', 'server.js', '-DSOCKK=%d' % self.listen_port] + self.args).communicate() - process = Popen(listify(NODE_JS) + ['server.js']) + process = Popen(NODE_JS + ['server.js']) self.pids.append(process.pid) def __exit__(self, *args, **kwargs): @@ -432,7 +432,7 @@ def test_webrtc(self): # note: you may need to run this manually yourself, if npm is not in the path, or if you need a version that is not in the path Popen(['npm', 'install', path_from_root('tests', 'sockets', 'p2p')]).communicate() - broker = Popen(listify(NODE_JS) + [path_from_root('tests', 'sockets', 'p2p', 'broker', 'p2p-broker.js')]) + broker = Popen(NODE_JS + [path_from_root('tests', 'sockets', 'p2p', 'broker', 'p2p-broker.js')]) expected = '1' self.run_browser(host_outfile, '.', ['/report_result?' + e for e in expected]) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 2ec01c2d8da74..24b9cee95e9a1 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -379,7 +379,6 @@ def sorter(x, y): return filename def run(filename, passes, js_engine=shared.NODE_JS, jcache=False, source_map=False, extra_info=None, just_split=False, just_concat=False): - js_engine = shared.listify(js_engine) return temp_files.run_and_clean(lambda: run_on_js(filename, passes, js_engine, jcache, source_map, extra_info, just_split, just_concat)) if __name__ == '__main__': diff --git a/tools/shared.py b/tools/shared.py index 92b57910f8df0..9ec88c19372be 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -6,10 +6,6 @@ import response_file import logging, platform, multiprocessing -def listify(x): - if type(x) is not list: return [x] - return x - # Temp file utilities from tempfiles import try_delete @@ -255,6 +251,7 @@ def new(*args): ============================================================================== ''' % (EM_CONFIG, CONFIG_FILE, llvm_root, node, __rootpath__) sys.exit(0) + try: config_text = open(CONFIG_FILE, 'r').read() if CONFIG_FILE else EM_CONFIG exec(config_text) @@ -262,6 +259,16 @@ def new(*args): logging.error('Error in evaluating %s (at %s): %s, text: %s' % (EM_CONFIG, CONFIG_FILE, str(e), config_text)) sys.exit(1) +def listify(x): + if type(x) is not list: return [x] + return x + +SPIDERMONKEY_ENGINE = listify(SPIDERMONKEY_ENGINE) +NODE_JS = listify(NODE_JS) +V8_ENGINE = listify(V8_ENGINE) +COMPILER_ENGINE = listify(COMPILER_ENGINE) +JS_ENGINES = [listify(ENGINE) for ENGINE in JS_ENGINES] + try: EM_POPEN_WORKAROUND except: @@ -344,8 +351,7 @@ def check_fastcomp(): def check_node_version(): try: - node = listify(NODE_JS) - actual = Popen(node + ['--version'], stdout=PIPE).communicate()[0].strip() + actual = Popen(NODE_JS + ['--version'], stdout=PIPE).communicate()[0].strip() version = tuple(map(int, actual.replace('v', '').replace('-pre', '').split('.'))) if version >= EXPECTED_NODE_VERSION: return True @@ -766,7 +772,7 @@ def include_directive(paths): try: if SPIDERMONKEY_ENGINE: - new_spidermonkey = listify(SPIDERMONKEY_ENGINE) + new_spidermonkey = SPIDERMONKEY_ENGINE if 'gcparam' not in str(new_spidermonkey): new_spidermonkey += ['-e', "gcparam('maxBytes', 1024*1024*1024);"] # Our very large files need lots of gc heap if '-w' not in str(new_spidermonkey): @@ -1600,7 +1606,7 @@ def pick_llvm_opts(optimization_level): @staticmethod def js_optimizer(filename, passes, jcache=False, debug=False, extra_info=None, output_filename=None, just_split=False, just_concat=False): - ret = js_optimizer.run(filename, passes, listify(NODE_JS), jcache, debug, extra_info, just_split, just_concat) + ret = js_optimizer.run(filename, passes, NODE_JS, jcache, debug, extra_info, just_split, just_concat) if output_filename: safe_move(ret, output_filename) ret = output_filename diff --git a/tools/validate_asmjs.py b/tools/validate_asmjs.py index 02ff506da7833..26bed51205b12 100644 --- a/tools/validate_asmjs.py +++ b/tools/validate_asmjs.py @@ -16,7 +16,7 @@ # Looks up SpiderMonkey engine using the variable SPIDERMONKEY_ENGINE in ~/.emscripten, and if not set up there, via PATH. def find_spidermonkey_engine(): - sm_engine = shared.listify(shared.SPIDERMONKEY_ENGINE) if hasattr(shared, 'SPIDERMONKEY_ENGINE') else [''] + sm_engine = shared.SPIDERMONKEY_ENGINE if hasattr(shared, 'SPIDERMONKEY_ENGINE') else [''] if not sm_engine or len(sm_engine[0]) == 0 or not os.path.exists(sm_engine[0]): sm_engine[0] = shared.Building.which('js') if sm_engine[0] == None: From 0960697c835b5d78f7ed92c70fd2bc906f1040b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Babi=C4=8Dka?= Date: Mon, 10 Nov 2014 18:17:06 +0100 Subject: [PATCH 073/161] Function take double instead of integer typedef void (* GLFWscrollfun)(GLFWwindow*,double,double); --- src/library_glfw.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library_glfw.js b/src/library_glfw.js index 0197d650d0e04..41e5d29c0110f 100644 --- a/src/library_glfw.js +++ b/src/library_glfw.js @@ -394,7 +394,7 @@ var LibraryGLFW = { sy = event.deltaY; } - Runtime.dynCall('viii', GLFW.active.scrollFunc, [GLFW.active.id, sx, sy]); + Runtime.dynCall('vidd', GLFW.active.scrollFunc, [GLFW.active.id, sx, sy]); #endif event.preventDefault(); From dd4580257e0ba6fdcaec164b45acc3703ff93d8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Babi=C4=8Dka?= Date: Mon, 10 Nov 2014 18:21:56 +0100 Subject: [PATCH 074/161] GLFW2 use same function callback as GLFW3 --- src/library_glfw.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library_glfw.js b/src/library_glfw.js index 41e5d29c0110f..d68ef98a5c29d 100644 --- a/src/library_glfw.js +++ b/src/library_glfw.js @@ -380,7 +380,7 @@ var LibraryGLFW = { if (!GLFW.active || !GLFW.active.scrollFunc || event.target != Module['canvas']) return; #if USE_GLFW == 2 - Runtime.dynCall('vi', GLFW.mouseWheelFunc, [GLFW.wheelPos]); + Runtime.dynCall('vi', GLFW.active.scrollFunc, [GLFW.wheelPos]); #endif #if USE_GLFW == 3 From f60bd1dd5b87ab74d950e80f2c43232ae78f4ea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Babi=C4=8Dka?= Date: Mon, 10 Nov 2014 18:23:26 +0100 Subject: [PATCH 075/161] Right name for mouse and scroll event handlers --- src/library_glfw.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/library_glfw.js b/src/library_glfw.js index d68ef98a5c29d..00300e583f7e8 100644 --- a/src/library_glfw.js +++ b/src/library_glfw.js @@ -487,13 +487,13 @@ var LibraryGLFW = { setCursorPosCallback: function(winid, cbfun) { var win = GLFW.WindowFromId(winid); if (!win) return; - win.mousePosFunc = cbfun; + win.cursorPosFunc = cbfun; }, setScrollCallback: function(winid, cbfun) { var win = GLFW.WindowFromId(winid); if (!win) return; - win.mouseWheelFunc = cbfun; + win.scrollFunc = cbfun; }, setWindowSizeCallback: function(winid, cbfun) { From 22138c3522bd8812ea467170110a8772b610b7c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Babi=C4=8Dka?= Date: Mon, 10 Nov 2014 18:27:22 +0100 Subject: [PATCH 076/161] Add Petr B. as author --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 6144947aac49e..50410d975a39d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,4 +1,4 @@ -The following authors have all licensed their contributions to Emscripten +The following authors have all licensed their contributions to Emscripten under the licensing terms detailed in LICENSE. (Authors keep copyright of their contributions, of course; they just grant @@ -168,4 +168,5 @@ a license to everyone to use it as detailed in LICENSE.) * William Furr * Dan Glastonbury (copyright owned by Mozilla Foundation) * Warren Seine (copyright owned by Aerys SAS) +* Petr Babicka From 2c68e408ad1871b24d80cbf35222da2c09dacb92 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 10 Nov 2014 13:25:55 -0800 Subject: [PATCH 077/161] initial work on string interning, and reusing the input for all strings --- tools/optimizer/minijson.h | 103 ++++++++++++++++++++++++++-------- tools/optimizer/optimizer.cpp | 51 ++++++++--------- 2 files changed, 105 insertions(+), 49 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index bffe14c552a08..350e7f09da9ba 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -16,10 +16,13 @@ #include #include #include +#include +#include #define err(str) fprintf(stderr, str "\n"); #define errv(str, ...) fprintf(stderr, str "\n", __VA_ARGS__); +struct IString; class Ref; struct Value; @@ -40,6 +43,8 @@ struct Ref { // special conveniences bool operator==(const char *str); // comparison to string, which is by value bool operator!=(const char *str); + bool operator==(const IString &str); + bool operator!=(const IString &str); bool operator==(double d) { assert(0); } // prevent Ref == number, which is potentially ambiguous; use ->getNumber() == number bool operator==(Ref other); bool operator!(); // check if null, in effect @@ -59,6 +64,58 @@ struct Arena { Arena arena; +// Interned String type, 100% interned on creation. Comparisons are always just a pointer comparison + +struct IString { + const char *str; + + class IStringHash : public std::hash { + public: + size_t operator()(const char *str) const { // TODO: optimize? + uint64_t ret = 0; + while (*str) { + ret = (ret*6364136223846793005ULL) + *str; + str++; + } + return (size_t)ret; + } + }; + class IStringEqual : public std::equal_to { + public: + bool operator()(const char * const &x, const char * const &y) const { + return strcmp(x, y) == 0; + } + }; + typedef std::unordered_set StringSet; + static StringSet strings; + + IString() : str(nullptr) {} + IString(const char *s) { + set(s); + } + + void set(const char *s) { + auto result = strings.insert(s); // if already present, does nothing + //errv("insert IString? %s : %d (size: %u)\n", s, result.second, strings.size()); + str = *(result.first); + } + + void set(const IString &s) { + str = s.str; + } + + bool operator==(const IString& other) { + assert((str == other.str) == !strcmp(str, other.str)); + return str == other.str; // fast! + } + bool operator!=(const IString& other) { + assert((str == other.str) == !strcmp(str, other.str)); + return str != other.str; // fast! + } +}; + +IString::StringSet IString::strings; + // Main value type struct Value { enum Type { @@ -74,7 +131,7 @@ struct Value { typedef std::vector ArrayStorage; union { // TODO: optimize - std::string *str; + IString str; double num; ArrayStorage *arr; bool boo; @@ -85,9 +142,6 @@ struct Value { explicit Value(const char *s) : type(Null) { setString(s); } - explicit Value(const std::string& s) : type(Null) { - setString(s.c_str()); - } explicit Value(double n) : type(Null) { setNumber(n); } @@ -102,8 +156,7 @@ struct Value { } void free() { - if (type == String) delete str; - else if (type == Array) delete arr; + if (type == Array) delete arr; type = Null; num = 0; } @@ -111,11 +164,14 @@ struct Value { Value& setString(const char *s) { free(); type = String; - str = new std::string(s); + str.set(s); return *this; } - Value& setString(const std::string& s) { - return setString(s.c_str()); + Value& setString(const IString &s) { + free(); + type = String; + str.set(s); + return *this; } Value& setNumber(double n) { free(); @@ -156,13 +212,9 @@ struct Value { bool isBool(bool b) { return type == Bool && b == boo; } // avoid overloading == as it might overload over int - std::string& getString() { - assert(isString()); - return *str; - } - const char* getCString() { + const char* getString() { assert(isString()); - return str->c_str(); + return str.str; } double& getNumber() { assert(isNumber()); @@ -181,7 +233,7 @@ struct Value { free(); switch (other.type) { case String: - setString(other.str->c_str()); + setString(other.str); break; case Number: setNumber(other.num); @@ -203,7 +255,7 @@ struct Value { if (type != other.type) return false; switch (other.type) { case String: - return *str == *(other.str); + return str == other.str; case Number: return num == other.num; case Array: @@ -237,9 +289,8 @@ struct Value { curr++; char *close = strchr(curr, '"'); assert(close); - *close = 0; + *close = 0; // end this string, and reuse it straight from the input setString(curr); - *close = '"'; curr = close+1; } else if (*curr == '[') { // Array @@ -286,7 +337,7 @@ struct Value { #define indentify() { for (int i = 0; i < indent; i++) os << " "; } switch (type) { case String: - os << '"' << *str << '"'; + os << '"' << str.str << '"'; break; case Number: os << std::setprecision(17) << num; // doubles can have 17 digits of precision @@ -393,11 +444,19 @@ Ref& Ref::operator[](unsigned x) { } bool Ref::operator==(const char *str) { - return get()->isString() && get()->getString() == str; + return get()->isString() && !strcmp(get()->str.str, str); } bool Ref::operator!=(const char *str) { - return get()->isString() ? get()->getString() != str : true; + return get()->isString() ? strcmp(get()->str.str, str) : true; +} + +bool Ref::operator==(const IString &str) { + return get()->isString() && get()->str == str; +} + +bool Ref::operator!=(const IString &str) { + return get()->isString() && get()->str != str; } bool Ref::operator==(Ref other) { diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 0d018a459eec1..3b87cb4c8eed5 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -4,8 +4,6 @@ #include #include -#include -#include #include "minijson.h" @@ -28,13 +26,12 @@ int parseInt(const char *str) { return ret; } -std::string getIntStr(int x) { +const char *getHeapStr(int x, bool unsign) { switch (x) { - case 1: return "1"; - case 8: return "8"; - case 16: return "16"; - case 32: return "32"; - case 64: return "64"; + case 8: return unsign ? "HEAPU8" : "HEAP8"; + case 16: return unsign ? "HEAPU16" : "HEAP16"; + case 32: return unsign ? "HEAPU32" : "HEAP32"; + case 64: return unsign ? "HEAPU64" : "HEAP64"; } assert(0); return ":("; @@ -128,7 +125,7 @@ void traverseFunctions(Ref ast, std::function visit) { } Ref deStat(Ref node) { - if (node[0]->getString() =="stat") return node[1]; + if (node[0] == "stat") return node[1]; return node; } @@ -206,7 +203,7 @@ struct AsmData { Ref name = node[2][1]; int index = func[2]->indexOf(name); if (index < 0) break; // not an assign into a parameter, but a global - std::string& str = name->getString(); + std::string str = name->getString(); if (locals.count(str) > 0) break; // already done that param, must be starting function body locals[str] = { detectType(node[3]), true }; params.push_back(locals.find(str)); @@ -284,7 +281,7 @@ struct AsmData { // add param coercions int next = 0; for (auto param : func[2]->getArray()) { - std::string& str = param->getString(); + std::string str = param->getString(); stats[next++] = make1("stat", make3("assign", &(arena.alloc())->setBool(true), makeName(str), makeAsmCoercion(makeName(str), locals[str].type))); } if (varDefs->size()) { @@ -303,7 +300,7 @@ struct AsmData { // ensure that there's a final "return" statement if needed. if (ret != ASM_NONE) { Ref retStmt = stats[stats->size() - 1]; - if (!retStmt || retStmt[0]->getString() != "return") { + if (!retStmt || retStmt[0] != "return") { Ref retVal = makeNum(0); if (ret != ASM_INT) { retVal = makeAsmCoercion(retVal, ret); @@ -383,7 +380,7 @@ AsmType detectType(Ref node, AsmData *asmData, bool inVarDef) { case 'c': { if (node[0] == "call") { if (node[1][0] == "name") { - std::string& name = node[1][1]->getString(); + std::string name = node[1][1]->getString(); if (name == "Math_fround") return ASM_FLOAT; else if (name == "SIMD_float32x4") return ASM_FLOAT32X4; else if (name == "SIMD_int32x4") return ASM_INT32X4; @@ -412,7 +409,7 @@ AsmType detectType(Ref node, AsmData *asmData, bool inVarDef) { return detectType(node[2], asmData, inVarDef); } else if (node[0] == "sub") { assert(node[1][0] == "name"); - HeapInfo info = parseHeap(node[1][1]->getCString()); + HeapInfo info = parseHeap(node[1][1]->getString()); if (info.valid) return ASM_NONE; return info.floaty ? ASM_DOUBLE : ASM_INT; // XXX ASM_FLOAT? } @@ -434,7 +431,7 @@ Ref makeString(const char *s) { } Ref makeString(const std::string& s) { - return &arena.alloc()->setString(s); + return &arena.alloc()->setString(s.c_str()); } Ref makeEmpty() { @@ -575,7 +572,7 @@ bool isMathFunc(const char *name) { } bool isMathFunc(Ref value) { - return value->isString() && isMathFunc(value->getString().c_str()); + return value->isString() && isMathFunc(value->getString()); } bool callHasSideEffects(Ref node) { // checks if the call itself (not the args) has side effects (or is not statically known) @@ -583,7 +580,7 @@ bool callHasSideEffects(Ref node) { // checks if the call itself (not the args) } bool hasSideEffects(Ref node) { // this is 99% incomplete! - std::string& type = node[0]->getString(); + std::string type = node[0]->getString(); switch (type[0]) { case 'n': if (type == "num" || type == "name") return false; @@ -623,7 +620,7 @@ Ref simplifyNotCompsDirect(Ref node) { // de-morgan's laws do not work on floats, due to nans >:( if (node[2][0] == "binary" && (detectType(node[2][2]) == ASM_INT && detectType(node[2][3]) == ASM_INT)) { Ref op = node[2][1]; - switch(op->getCString()[0]) { + switch(op->getString()[0]) { case '<': { if (op == "<") { op->setString(">="); break; } if (op == "<=") { op->setString(">"); break; } @@ -708,7 +705,7 @@ class StringSet : public std::unordered_set { return count(str) > 0; } bool has(Ref node) { - return has(node->getString().c_str()); + return has(node->getString()); } }; @@ -726,7 +723,7 @@ bool isFunctionTable(const char *name) { } bool isFunctionTable(Ref value) { - return value->isString() && isFunctionTable(value->getString().c_str()); + return value->isString() && isFunctionTable(value->getString()); } void simplifyExpressions(Ref ast) { @@ -868,11 +865,11 @@ void simplifyExpressions(Ref ast) { node[2] = input[2]; } else if (input[0] == "sub" && input[1][0] == "name") { // HEAP8[..] & 255 => HEAPU8[..] - HeapInfo hi = parseHeap(input[1][1]->getCString()); + HeapInfo hi = parseHeap(input[1][1]->getString()); if (hi.valid) { if (isInteger32(amount) && amount == powl(2, hi.bits)-1) { if (!hi.unsign) { - input[1][1]->setString("HEAPU" + getIntStr(hi.bits)); // make unsigned + input[1][1]->setString(getHeapStr(hi.bits, true)); // make unsigned } // we cannot return HEAPU8 without a coercion, but at least we do HEAP8 & 255 => HEAPU8 | 0 node[1]->setString("|"); @@ -895,9 +892,9 @@ void simplifyExpressions(Ref ast) { // collapse HEAPU?8[..] << 24 >> 24 etc. into HEAP8[..] | 0 double amount = node[3][1]->getNumber(); if (amount == node[2][3][1]->getNumber()) { - HeapInfo hi = parseHeap(node[2][2][1][1]->getCString()); + HeapInfo hi = parseHeap(node[2][2][1][1]->getString()); if (hi.valid && hi.bits == 32 - amount) { - node[2][2][1][1]->setString("HEAP" + getIntStr(hi.bits)); + node[2][2][1][1]->setString(getHeapStr(hi.bits, false)); node[1]->setString("|"); node[2] = node[2][2]; node[3][1]->setNumber(0); @@ -1080,7 +1077,7 @@ void simplifyExpressions(Ref ast) { // do we want a simplifybitops on the new values here? } for (auto use : info.uses) { - use[2][1][1]->setString(correct); + use[2][1][1]->setString(correct.c_str()); } AsmType correctType; switch(asmData.getType(v)) { @@ -1096,7 +1093,7 @@ void simplifyExpressions(Ref ast) { }; std::function emitsBoolean = [&emitsBoolean](Ref node) { - std::string& type = node[0]->getString(); + std::string type = node[0]->getString(); if (type == "num") { return node[1]->getNumber() == 0 || node[1]->getNumber() == 1; } @@ -1373,7 +1370,7 @@ int main(int argc, char **argv) { // Parse JSON source into the document doc = arena.alloc(); doc->parse(json); - delete[] json; + // do not free json, it's contents are used as strings // Run passes on the Document for (int i = 2; i < argc; i++) { From 89b200bb02fef67b4896398cd74fb333da506710 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 10 Nov 2014 13:46:58 -0800 Subject: [PATCH 078/161] remove unneeded include --- tools/optimizer/minijson.h | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 350e7f09da9ba..85296e869e568 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -7,7 +7,6 @@ //#define NDEBUG #include -#include #include #include From 97d68a3420d7769d217954d88e19ef1c528afddd Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 10 Nov 2014 14:29:29 -0800 Subject: [PATCH 079/161] intern all common strings --- tools/optimizer/minijson.h | 14 +- tools/optimizer/optimizer.cpp | 536 ++++++++++++++++++---------------- 2 files changed, 304 insertions(+), 246 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 85296e869e568..ef05aa8dab06c 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -111,6 +111,14 @@ struct IString { assert((str == other.str) == !strcmp(str, other.str)); return str != other.str; // fast! } + + char operator[](int x) { + return str[x]; + } + + const char *c_str() { return str; } + + bool isNull() { return str == nullptr; } }; IString::StringSet IString::strings; @@ -211,10 +219,14 @@ struct Value { bool isBool(bool b) { return type == Bool && b == boo; } // avoid overloading == as it might overload over int - const char* getString() { + const char* getCString() { assert(isString()); return str.str; } + IString& getIString() { + assert(isString()); + return str; + } double& getNumber() { assert(isNumber()); return num; diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 3b87cb4c8eed5..78a6658899402 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -13,6 +13,60 @@ Ref doc; +IString TOPLEVEL("toplevel"), + DEFUN("defun"), + BLOCK("block"), + STAT("stat"), + ASSIGN("assign"), + NAME("name"), + VAR("var"), + CONDITIONAL("conditional"), + BINARY("binary"), + RETURN("return"), + IF("if"), + WHILE("while"), + SEQ("seq"), + SUB("sub"), + CALL("call"), + NUM("num"), + LABEL("label"), + STRING("string"), + INF("inf"), + NaN("nan"), + TEMP_RET0("tempRet0"), + UNARY_PREFIX("unary-prefix"), + MATH_FROUND("Math_fround"), + SIMD_FLOAT32X4("SIMD_float32x4"), + SIMD_INT32X4("SIMD_int32x4"), + PLUS("+"), + MINUS("-"), + OR("|"), + AND("&"), + XOR("^"), + L_NOT("!"), + B_NOT("~"), + LT("<"), + GE(">="), + LE("<="), + GT(">"), + EQ("=="), + NE("!="), + DIV("/"), + MOD("%"), + RSHIFT(">>"), + LSHIFT("<<"), + TRSHIFT(">>>"), + TEMP_DOUBLE_PTR("tempDoublePtr"), + HEAP8("HEAP8"), + HEAP16("HEAP16"), + HEAP32("HEAP32"), + HEAPF32("HEAPF32"), + HEAPU8("HEAPU8"), + HEAPU16("HEAPU16"), + HEAPU32("HEAPU32"), + HEAPF64("HEAPF64"), + F0("f0"); + //================== // Infrastructure //================== @@ -26,12 +80,11 @@ int parseInt(const char *str) { return ret; } -const char *getHeapStr(int x, bool unsign) { +IString getHeapStr(int x, bool unsign) { switch (x) { - case 8: return unsign ? "HEAPU8" : "HEAP8"; - case 16: return unsign ? "HEAPU16" : "HEAP16"; - case 32: return unsign ? "HEAPU32" : "HEAP32"; - case 64: return unsign ? "HEAPU64" : "HEAP64"; + case 8: return unsign ? HEAPU8 : HEAP8; + case 16: return unsign ? HEAPU16 : HEAP16; + case 32: return unsign ? HEAPU32 : HEAP32; } assert(0); return ":("; @@ -113,26 +166,26 @@ void traversePrePostConditional(Ref node, std::function visitPre, st // Traverses all the top-level functions in the document void traverseFunctions(Ref ast, std::function visit) { - if (ast[0] == "toplevel") { + if (ast[0] == TOPLEVEL) { Ref stats = ast[1]; for (int i = 0; i < stats->size(); i++) { Ref curr = stats[i]; - if (curr[0] == "defun") visit(curr); + if (curr[0] == DEFUN) visit(curr); } - } else if (ast[0] == "defun") { + } else if (ast[0] == DEFUN) { visit(ast); } } Ref deStat(Ref node) { - if (node[0] == "stat") return node[1]; + if (node[0] == STAT) return node[1]; return node; } Ref getStatements(Ref node) { - if (node[0] == "defun") { + if (node[0] == DEFUN) { return node[3]; - } else if (node[0] == "block") { + } else if (node[0] == BLOCK) { return node[1]; } else { return arena.alloc(); @@ -158,10 +211,10 @@ bool isEmpty(Ref node); Ref makeAsmVarDef(const std::string& v_, AsmType type); Ref makeArray(); Ref makeNum(double x); -Ref makeName(const std::string& str); +Ref makeName(IString str); Ref makeAsmCoercion(Ref node, AsmType type); -Ref make1(const char* type, Ref a); -Ref make3(const char *type, Ref a, Ref b, Ref c); +Ref make1(IString type, Ref a); +Ref make3(IString type, Ref a, Ref b, Ref c); struct AsmData { struct Local { @@ -198,12 +251,12 @@ struct AsmData { int i = 0; while (i < stats->size()) { Ref node = stats[i]; - if (node[0] != "stat" || node[1][0] != "assign" || node[1][2][0] != "name") break; + if (node[0] != STAT || node[1][0] != ASSIGN || node[1][2][0] != NAME) break; node = node[1]; Ref name = node[2][1]; int index = func[2]->indexOf(name); if (index < 0) break; // not an assign into a parameter, but a global - std::string str = name->getString(); + std::string str = name->getCString(); if (locals.count(str) > 0) break; // already done that param, must be starting function body locals[str] = { detectType(node[3]), true }; params.push_back(locals.find(str)); @@ -213,10 +266,10 @@ struct AsmData { // process initial variable definitions while (i < stats->size()) { Ref node = stats[i]; - if (node[0] != "var") break; + if (node[0] != VAR) break; for (int j = 0; j < node[1]->size(); j++) { Ref v = node[1][j]; - std::string name = v[0]->getString(); + std::string name = v[0]->getCString(); Ref value = v[1]; if (locals.count(name) == 0) { locals[name] = { detectType(value, nullptr, true), false }; @@ -234,16 +287,16 @@ struct AsmData { while (i < stats->size()) { traversePre(stats[i], [&](Ref node) { Ref type = node[0]; - if (type == "var") { + if (type == VAR) { dump("bad, seeing a var in need of fixing", func); assert(0); //, 'should be no vars to fix! ' + func[1] + ' : ' + JSON.stringify(node)); } }); i++; } - // look for final "return" statement to get return type. + // look for final RETURN statement to get return type. Ref retStmt = stats[stats->size() - 1]; - if (!!retStmt && retStmt[0] == "return" && !!retStmt[1]) { + if (!!retStmt && retStmt[0] == RETURN && !!retStmt[1]) { ret = detectType(retStmt[1]); } else { ret = ASM_NONE; @@ -254,7 +307,7 @@ struct AsmData { Ref stats = func[3]; // Remove var definitions, if any for (int i = 0; i < stats->size(); i++) { - if (stats[i][0] == "var") { + if (stats[i][0] == VAR) { stats[i] = makeEmpty(); } else { if (!isEmpty(stats[i])) break; @@ -281,31 +334,31 @@ struct AsmData { // add param coercions int next = 0; for (auto param : func[2]->getArray()) { - std::string str = param->getString(); - stats[next++] = make1("stat", make3("assign", &(arena.alloc())->setBool(true), makeName(str), makeAsmCoercion(makeName(str), locals[str].type))); + std::string str = param->getCString(); + stats[next++] = make1(STAT, make3(ASSIGN, &(arena.alloc())->setBool(true), makeName(str.c_str()), makeAsmCoercion(makeName(str.c_str()), locals[str].type))); } if (varDefs->size()) { - stats[next] = make1("var", varDefs); + stats[next] = make1(VAR, varDefs); } /* if (inlines.length > 0) { var i = 0; traverse(func, function(node, type) { - if (type == "call" && node[1][0] == "name" && node[1][1] == 'inlinejs') { + if (type == CALL && node[1][0] == NAME && node[1][1] == 'inlinejs') { node[1] = inlines[i++]; // swap back in the body } }); } */ - // ensure that there's a final "return" statement if needed. + // ensure that there's a final RETURN statement if needed. if (ret != ASM_NONE) { Ref retStmt = stats[stats->size() - 1]; - if (!retStmt || retStmt[0] != "return") { + if (!retStmt || retStmt[0] != RETURN) { Ref retVal = makeNum(0); if (ret != ASM_INT) { retVal = makeAsmCoercion(retVal, ret); } - stats->push_back(make1("return", retVal)); + stats->push_back(make1(RETURN, retVal)); } } //printErr('denormalized \n\n' + astToSrc(func) + '\n\n'); @@ -341,34 +394,34 @@ bool isInteger32(double x) { return isInteger(x) && (x == (int32_t)x || x == (uint32_t)x); } -std::string ASM_FLOAT_ZERO; +IString ASM_FLOAT_ZERO; AsmType detectType(Ref node, AsmData *asmData, bool inVarDef) { - switch (node[0]->getString()[0]) { + switch (node[0]->getCString()[0]) { case 'n': { - if (node[0] == "num") { + if (node[0] == NUM) { if (!isInteger(node[1]->getNumber())) return ASM_DOUBLE; return ASM_INT; - } else if (node[0] == "name") { + } else if (node[0] == NAME) { if (asmData) { - AsmType ret = asmData->getType(node[1]->getString()); + AsmType ret = asmData->getType(node[1]->getCString()); if (ret != ASM_NONE) return ret; } if (!inVarDef) { - if (node[1] == "inf" || node[1] == "nan") return ASM_DOUBLE; - if (node[1] == "tempRet0") return ASM_INT; + if (node[1] == INF || node[1] == NaN) return ASM_DOUBLE; + if (node[1] == TEMP_RET0) return ASM_INT; return ASM_NONE; } // We are in a variable definition, where Math_fround(0) optimized into a global constant becomes f0 = Math_fround(0) - if (ASM_FLOAT_ZERO.size() == 0) ASM_FLOAT_ZERO = node[1]->getString(); - else assert(ASM_FLOAT_ZERO == node[1]->getString()); + if (ASM_FLOAT_ZERO.isNull()) ASM_FLOAT_ZERO = node[1]->getIString(); + else assert(node[1] == ASM_FLOAT_ZERO); return ASM_FLOAT; } break; } case 'u': { - if (node[0] == "unary-prefix") { - switch (node[1]->getString()[0]) { + if (node[0] == UNARY_PREFIX) { + switch (node[1]->getCString()[0]) { case '+': return ASM_DOUBLE; case '-': return detectType(node[2], asmData, inVarDef); case '!': case '~': return ASM_INT; @@ -378,22 +431,22 @@ AsmType detectType(Ref node, AsmData *asmData, bool inVarDef) { break; } case 'c': { - if (node[0] == "call") { - if (node[1][0] == "name") { - std::string name = node[1][1]->getString(); - if (name == "Math_fround") return ASM_FLOAT; - else if (name == "SIMD_float32x4") return ASM_FLOAT32X4; - else if (name == "SIMD_int32x4") return ASM_INT32X4; + if (node[0] == CALL) { + if (node[1][0] == NAME) { + IString name = node[1][1]->getIString(); + if (name == MATH_FROUND) return ASM_FLOAT; + else if (name == SIMD_FLOAT32X4) return ASM_FLOAT32X4; + else if (name == SIMD_INT32X4) return ASM_INT32X4; } return ASM_NONE; - } else if (node[0] == "conditional") { + } else if (node[0] == CONDITIONAL) { return detectType(node[2], asmData, inVarDef); } break; } case 'b': { - if (node[0] == "binary") { - switch (node[1]->getString()[0]) { + if (node[0] == BINARY) { + switch (node[1]->getCString()[0]) { case '+': case '-': case '*': case '/': case '%': return detectType(node[2], asmData, inVarDef); case '|': case '&': case '^': case '<': case '>': // handles <<, >>, >>=, <=, >= @@ -405,11 +458,11 @@ AsmType detectType(Ref node, AsmData *asmData, bool inVarDef) { break; } case 's': { - if (node[0] == "seq") { + if (node[0] == SEQ) { return detectType(node[2], asmData, inVarDef); - } else if (node[0] == "sub") { - assert(node[1][0] == "name"); - HeapInfo info = parseHeap(node[1][1]->getString()); + } else if (node[0] == SUB) { + assert(node[1][0] == NAME); + HeapInfo info = parseHeap(node[1][1]->getCString()); if (info.valid) return ASM_NONE; return info.floaty ? ASM_DOUBLE : ASM_INT; // XXX ASM_FLOAT? } @@ -426,17 +479,13 @@ Ref makeArray() { return &arena.alloc()->setArray(); } -Ref makeString(const char *s) { +Ref makeString(const IString& s) { return &arena.alloc()->setString(s); } -Ref makeString(const std::string& s) { - return &arena.alloc()->setString(s.c_str()); -} - Ref makeEmpty() { Ref ret(makeArray()); - ret->push_back(makeString("toplevel")); + ret->push_back(makeString(TOPLEVEL)); ret->push_back(makeArray()); return ret; } @@ -450,37 +499,33 @@ Ref makePair(Ref x, Ref y) { Ref makeNum(double x) { Ref ret(makeArray()); - ret->push_back(makeString("num")); + ret->push_back(makeString(NUM)); ret->push_back(&arena.alloc()->setNumber(x)); return ret; } -Ref makeName(const char *str) { +Ref makeName(IString str) { Ref ret(makeArray()); - ret->push_back(makeString("name")); + ret->push_back(makeString(NAME)); ret->push_back(makeString(str)); return ret; } -Ref makeName(const std::string& str) { - return makeName(str.c_str()); -} - Ref makeBlock() { Ref ret(makeArray()); - ret->push_back(makeString("block")); + ret->push_back(makeString(BLOCK)); ret->push_back(makeArray()); return ret; } -Ref make1(const char* type, Ref a) { +Ref make1(IString type, Ref a) { Ref ret(makeArray()); ret->push_back(makeString(type)); ret->push_back(a); return ret; } -Ref make2(const char* type, const char *a, Ref b) { +Ref make2(IString type, IString a, Ref b) { Ref ret(makeArray()); ret->push_back(makeString(type)); ret->push_back(makeString(a)); @@ -488,7 +533,7 @@ Ref make2(const char* type, const char *a, Ref b) { return ret; } -Ref make2(const char* type, Ref a, Ref b) { +Ref make2(IString type, Ref a, Ref b) { Ref ret(makeArray()); ret->push_back(makeString(type)); ret->push_back(a); @@ -496,7 +541,7 @@ Ref make2(const char* type, Ref a, Ref b) { return ret; } -Ref make3(const char *type, const char *a, Ref b, Ref c) { +Ref make3(IString type, IString a, Ref b, Ref c) { Ref ret(makeArray()); ret->push_back(makeString(type)); ret->push_back(makeString(a)); @@ -505,7 +550,7 @@ Ref make3(const char *type, const char *a, Ref b, Ref c) { return ret; } -Ref make3(const char *type, Ref a, Ref b, Ref c) { +Ref make3(IString type, Ref a, Ref b, Ref c) { Ref ret(makeArray()); ret->push_back(makeString(type)); ret->push_back(a); @@ -515,25 +560,25 @@ Ref make3(const char *type, Ref a, Ref b, Ref c) { } Ref makeAsmVarDef(const std::string& v_, AsmType type) { - Ref v = makeString(v_); + Ref v = makeString(IString(v_.c_str())); Ref val; switch (type) { case ASM_INT: val = makeNum(0); break; - case ASM_DOUBLE: val = make2("unary-prefix", "+", makeNum(0)); break; + case ASM_DOUBLE: val = make2(UNARY_PREFIX, PLUS, makeNum(0)); break; case ASM_FLOAT: { - if (ASM_FLOAT_ZERO.size() > 0) { - val = makeName(ASM_FLOAT_ZERO.c_str()); + if (!ASM_FLOAT_ZERO.isNull()) { + val = makeName(ASM_FLOAT_ZERO); } else { - val = make2("call", makeName("Math_fround"), &(makeArray())->push_back(makeNum(0))); + val = make2(CALL, makeName(MATH_FROUND), &(makeArray())->push_back(makeNum(0))); } break; } case ASM_FLOAT32X4: { - val = make2("call", makeName("SIMD_float32x4"), &(makeArray())->push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0))); + val = make2(CALL, makeName(SIMD_FLOAT32X4), &(makeArray())->push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0))); break; } case ASM_INT32X4: { - val = make2("call", makeName("SIMD_int32x4"), &(makeArray())->push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0))); + val = make2(CALL, makeName(SIMD_INT32X4), &(makeArray())->push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0)).push_back(makeNum(0))); break; } default: assert(0); @@ -543,11 +588,11 @@ Ref makeAsmVarDef(const std::string& v_, AsmType type) { Ref makeAsmCoercion(Ref node, AsmType type) { switch (type) { - case ASM_INT: return make3("binary", "|", node, makeNum(0)); - case ASM_DOUBLE: return make2("unary-prefix", "+", node); - case ASM_FLOAT: return make2("call", makeName("Math_fround"), &(makeArray())->push_back(node)); - case ASM_FLOAT32X4: return make2("call", makeName("SIMD_float32x4"), &(makeArray())->push_back(node)); - case ASM_INT32X4: return make2("call", makeName("SIMD_int32x4"), &(makeArray())->push_back(node)); + case ASM_INT: return make3(BINARY, OR, node, makeNum(0)); + case ASM_DOUBLE: return make2(UNARY_PREFIX, PLUS, node); + case ASM_FLOAT: return make2(CALL, makeName(MATH_FROUND), &(makeArray())->push_back(node)); + case ASM_FLOAT32X4: return make2(CALL, makeName(SIMD_FLOAT32X4), &(makeArray())->push_back(node)); + case ASM_INT32X4: return make2(CALL, makeName(SIMD_INT32X4), &(makeArray())->push_back(node)); case ASM_NONE: default: return node; // non-validating code, emit nothing XXX this is dangerous, we should only allow this when we know we are not validating } @@ -556,12 +601,12 @@ Ref makeAsmCoercion(Ref node, AsmType type) { // Checks bool isEmpty(Ref node) { - return node->size() == 2 && node[0] == "toplevel" && node[1]->size() == 0; + return node->size() == 2 && node[0] == TOPLEVEL && node[1]->size() == 0; } bool commable(Ref node) { // TODO: hashing - std::string type = node[0]->getString(); - if (type == "assign" || type == "binary" || type == "unary-prefix" || type == "unary-postfix" || type == "name" || type == "num" || type == "call" || type == "seq" || type == "conditional" || type == "sub") return true; + IString type = node[0]->getIString(); + if (type == ASSIGN || type == BINARY || type == UNARY_PREFIX || type == NAME || type == NUM || type == CALL || type == SEQ || type == CONDITIONAL || type == SUB) return true; return false; } @@ -572,38 +617,38 @@ bool isMathFunc(const char *name) { } bool isMathFunc(Ref value) { - return value->isString() && isMathFunc(value->getString()); + return value->isString() && isMathFunc(value->getCString()); } bool callHasSideEffects(Ref node) { // checks if the call itself (not the args) has side effects (or is not statically known) - return !(node[1][0] == "name" && isMathFunc(node[1][1])); + return !(node[1][0] == NAME && isMathFunc(node[1][1])); } bool hasSideEffects(Ref node) { // this is 99% incomplete! - std::string type = node[0]->getString(); + IString type = node[0]->getIString(); switch (type[0]) { case 'n': - if (type == "num" || type == "name") return false; + if (type == NUM || type == NAME) return false; break; case 's': - if (type == "string") return false; - if (type == "sub") return hasSideEffects(node[1]) || hasSideEffects(node[2]); + if (type == STRING) return false; + if (type == SUB) return hasSideEffects(node[1]) || hasSideEffects(node[2]); break; case 'u': - if (type == "unary-prefix") return hasSideEffects(node[2]); + if (type == UNARY_PREFIX) return hasSideEffects(node[2]); break; case 'b': - if (type == "binary") return hasSideEffects(node[2]) || hasSideEffects(node[3]); + if (type == BINARY) return hasSideEffects(node[2]) || hasSideEffects(node[3]); break; case 'c': - if (type == "call") { + if (type == CALL) { if (callHasSideEffects(node)) return true; // This is a statically known call, with no side effects. only args can side effect us for (auto arg : node[2]->getArray()) { if (hasSideEffects(arg)) return true; } return false; - } else if (type == "conditional") return hasSideEffects(node[1]) || hasSideEffects(node[2]) || hasSideEffects(node[3]); + } else if (type == CONDITIONAL) return hasSideEffects(node[1]) || hasSideEffects(node[2]) || hasSideEffects(node[3]); break; } return true; @@ -616,33 +661,33 @@ bool hasSideEffects(Ref node) { // this is 99% incomplete! // if (!(x < 5)) // or such. Simplifying these saves space and time. Ref simplifyNotCompsDirect(Ref node) { - if (node[0] == "unary-prefix" && node[1] == "!") { + if (node[0] == UNARY_PREFIX && node[1] == L_NOT) { // de-morgan's laws do not work on floats, due to nans >:( - if (node[2][0] == "binary" && (detectType(node[2][2]) == ASM_INT && detectType(node[2][3]) == ASM_INT)) { + if (node[2][0] == BINARY && (detectType(node[2][2]) == ASM_INT && detectType(node[2][3]) == ASM_INT)) { Ref op = node[2][1]; - switch(op->getString()[0]) { + switch(op->getCString()[0]) { case '<': { - if (op == "<") { op->setString(">="); break; } - if (op == "<=") { op->setString(">"); break; } + if (op == LT) { op->setString(GE); break; } + if (op == LE) { op->setString(GE); break; } return node; } case '>': { - if (op == ">") { op->setString("<="); break; } - if (op == ">=") { op->setString("<"); break; } + if (op == GE) { op->setString(LE); break; } + if (op == GE) { op->setString(LT); break; } return node; } case '=': { - if (op == "==") { op->setString("!="); break; } + if (op == EQ) { op->setString(NE); break; } return node; } case '!': { - if (op == "!=") { op->setString("=="); break; } + if (op == NE) { op->setString(EQ); break; } return node; } default: return node; } - return make3("binary", op, node[2][2], node[2][3]); - } else if (node[2][0] == "unary-prefix" && node[2][1] == "!") { + return make3(BINARY, op, node[2][2], node[2][3]); + } else if (node[2][0] == UNARY_PREFIX && node[2][1] == L_NOT) { return node[2][2]; } } @@ -650,7 +695,7 @@ Ref simplifyNotCompsDirect(Ref node) { } Ref flipCondition(Ref cond) { - return simplifyNotCompsDirect(make2("unary-prefix", "!", cond)); + return simplifyNotCompsDirect(make2(UNARY_PREFIX, L_NOT, cond)); } void safeCopy(Ref target, Ref source) { // safely copy source onto target, even if source is a subnode of target @@ -664,13 +709,13 @@ int measureCost(Ref ast) { int size = 0; traversePre(ast, [&size](Ref node) { Ref type = node[0]; - if (type == "num" || type == "unary-prefix") size--; - else if (type == "binary") { - if (node[3][0] == "num" && node[3][1]->getNumber() == 0) size--; - else if (node[1] == "/" || node[1] == "%") size += 2; + if (type == NUM || type == UNARY_PREFIX) size--; + else if (type == BINARY) { + if (node[3][0] == NUM && node[3][1]->getNumber() == 0) size--; + else if (node[1] == DIV || node[1] == MOD) size += 2; } - else if (type == "call" && !callHasSideEffects(node)) size -= 2; - else if (type == "sub") size++; + else if (type == CALL && !callHasSideEffects(node)) size -= 2; + else if (type == SUB) size++; size++; }); return size; @@ -705,7 +750,7 @@ class StringSet : public std::unordered_set { return count(str) > 0; } bool has(Ref node) { - return has(node->getString()); + return has(node->getCString()); } }; @@ -723,7 +768,7 @@ bool isFunctionTable(const char *name) { } bool isFunctionTable(Ref value) { - return value->isString() && isFunctionTable(value->getString()); + return value->isString() && isFunctionTable(value->getCString()); } void simplifyExpressions(Ref ast) { @@ -732,27 +777,27 @@ void simplifyExpressions(Ref ast) { auto simplifyIntegerConversions = [](Ref ast) { traversePre(ast, [](Ref node) { Ref type = node[0]; - if (type == "binary" && node[1] == ">>" && node[3][0] == "num" && - node[2][0] == "binary" && node[2][1] == "<<" && node[2][3][0] == "num" && node[3][1]->getNumber() == node[2][3][1]->getNumber()) { + if (type == BINARY && node[1] == RSHIFT && node[3][0] == NUM && + node[2][0] == BINARY && node[2][1] == LSHIFT && node[2][3][0] == NUM && node[3][1]->getNumber() == node[2][3][1]->getNumber()) { // Transform (x&A)<>B to X&A. Ref innerNode = node[2][2]; double shifts = node[3][1]->getNumber(); - if (innerNode[0] == "binary" && innerNode[1] == "&" && innerNode[3][0] == "num") { + if (innerNode[0] == BINARY && innerNode[1] == AND && innerNode[3][0] == NUM) { double mask = innerNode[3][1]->getNumber(); if (isInteger32(mask) && isInteger32(shifts) && ((int(mask) << int(shifts)) >> int(shifts)) == int(mask)) { safeCopy(node, innerNode); return; } } - } else if (type == "binary" && BITWISE.has(node[1])) { + } else if (type == BINARY && BITWISE.has(node[1])) { for (int i = 2; i <= 3; i++) { Ref subNode = node[i]; - if (subNode[0] == "binary" && subNode[1] == "&" && subNode[3][0] == "num" && subNode[3][1]->getNumber() == 1) { + if (subNode[0] == BINARY && subNode[1] == AND && subNode[3][0] == NUM && subNode[3][1]->getNumber() == 1) { // Rewrite (X < Y) & 1 to X < Y , when it is going into a bitwise operator. We could // remove even more (just replace &1 with |0, then subsequent passes could remove the |0) // but v8 issue #2513 means the code would then run very slowly in chrome. Ref input = subNode[2]; - if (input[0] == "binary" && COMPARE_OPS.has(input[1])) { + if (input[0] == BINARY && COMPARE_OPS.has(input[1])) { safeCopy(node[i], input); } } @@ -775,21 +820,21 @@ void simplifyExpressions(Ref ast) { std::vector stack; std::function process = [&stack, &rerun, &process, &ast](Ref node) { Ref type = node[0]; - if (type == "binary" && node[1] == "|") { - if (node[2][0] == "num" && node[3][0] == "num") { + if (type == BINARY && node[1] == OR) { + if (node[2][0] == NUM && node[3][0] == NUM) { node[2][1]->setNumber(int(node[2][1]->getNumber()) | int(node[3][1]->getNumber())); stack.push_back(0); safeCopy(node, node[2]); return; } bool go = false; - if (node[2][0] == "num" && node[2][1]->getNumber() == 0) { + if (node[2][0] == NUM && node[2][1]->getNumber() == 0) { // canonicalize order Ref temp = node[3]; node[3] = node[2]; node[2] = temp; go = true; - } else if (node[3][0] == "num" && node[3][1]->getNumber() == 0) { + } else if (node[3][0] == NUM && node[3][1]->getNumber() == 0) { go = true; } if (!go) { @@ -799,11 +844,11 @@ void simplifyExpressions(Ref ast) { // We might be able to remove this correction for (int i = stack.size()-1; i >= 0; i--) { if (stack[i] >= 1) { - if (stack[stack.size()-1] < 2 && node[2][0] == "call") break; // we can only remove multiple |0s on these + if (stack[stack.size()-1] < 2 && node[2][0] == CALL) break; // we can only remove multiple |0s on these if (stack[stack.size()-1] < 1 && (COERCION_REQUIRING_OPS.has(node[2][0]) || - (node[2][0] == "binary" && COERCION_REQUIRING_BINARIES.has(node[2][1])))) break; // we can remove |0 or >>2 + (node[2][0] == BINARY && COERCION_REQUIRING_BINARIES.has(node[2][1])))) break; // we can remove |0 or >>2 // we will replace ourselves with the non-zero side. Recursively process that node. - Ref result = node[2][0] == "num" && node[2][1]->getNumber() == 0 ? node[3] : node[2], other; + Ref result = node[2][0] == NUM && node[2][1]->getNumber() == 0 ? node[3] : node[2], other; // replace node in-place safeCopy(node, result); rerun = true; @@ -815,11 +860,11 @@ void simplifyExpressions(Ref ast) { } stack.push_back(2); // From here on up, no need for this kind of correction, it's done at the top // (Add this at the end, so it is only added if we did not remove it) - } else if (type == "binary" && USEFUL_BINARY_OPS.has(node[1])) { + } else if (type == BINARY && USEFUL_BINARY_OPS.has(node[1])) { stack.push_back(1); - } else if ((type == "binary" && SAFE_BINARY_OPS.has(node[1])) || type == "num" || type == "name") { + } else if ((type == BINARY && SAFE_BINARY_OPS.has(node[1])) || type == NUM || type == NAME) { stack.push_back(0); // This node is safe in that it does not interfere with this optimization - } else if (type == "unary-prefix" && node[1] == "~") { + } else if (type == UNARY_PREFIX && node[1] == B_NOT) { stack.push_back(1); } else { stack.push_back(-1); // This node is dangerous! Give up if you see this before you see '1' @@ -841,7 +886,7 @@ void simplifyExpressions(Ref ast) { traversePrePostConditional(ast, [](Ref node) { // Detect trees which should not // be simplified. - if (node[0] == "sub" && node[1][0] == "name" && isFunctionTable(node[1][1])) { + if (node[0] == SUB && node[1][0] == NAME && isFunctionTable(node[1][1])) { return false; // do not traverse subchildren here, we should not collapse 55 & 126. } return true; @@ -850,106 +895,106 @@ void simplifyExpressions(Ref ast) { // that we simplify a node's operands before the node itself. This allows // optimizations to cascade. Ref type = node[0]; - if (type == "name") { - if (node[1] == "tempDoublePtr") hasTempDoublePtr = true; - } else if (type == "binary" && node[1] == "&" && node[3][0] == "num") { - if (node[2][0] == "num") { + if (type == NAME) { + if (node[1] == TEMP_DOUBLE_PTR) hasTempDoublePtr = true; + } else if (type == BINARY && node[1] == AND && node[3][0] == NUM) { + if (node[2][0] == NUM) { safeCopy(node, makeNum(int(node[2][1]->getNumber()) & int(node[3][1]->getNumber()))); return; } Ref input = node[2]; double amount = node[3][1]->getNumber(); - if (input[0] == "binary" && input[1] == "&" && input[3][0] == "num") { + if (input[0] == BINARY && input[1] == AND && input[3][0] == NUM) { // Collapse X & 255 & 1 node[3][1]->setNumber(int(amount) & int(input[3][1]->getNumber())); node[2] = input[2]; - } else if (input[0] == "sub" && input[1][0] == "name") { + } else if (input[0] == SUB && input[1][0] == NAME) { // HEAP8[..] & 255 => HEAPU8[..] - HeapInfo hi = parseHeap(input[1][1]->getString()); + HeapInfo hi = parseHeap(input[1][1]->getCString()); if (hi.valid) { if (isInteger32(amount) && amount == powl(2, hi.bits)-1) { if (!hi.unsign) { input[1][1]->setString(getHeapStr(hi.bits, true)); // make unsigned } // we cannot return HEAPU8 without a coercion, but at least we do HEAP8 & 255 => HEAPU8 | 0 - node[1]->setString("|"); + node[1]->setString(OR); node[3][1]->setNumber(0); return; } } } - } else if (type == "binary" && node[1] == "^") { + } else if (type == BINARY && node[1] == XOR) { // LLVM represents bitwise not as xor with -1. Translate it back to an actual bitwise not. - if (node[3][0] == "unary-prefix" && node[3][1] == "-" && node[3][2][0] == "num" && + if (node[3][0] == UNARY_PREFIX && node[3][1] == MINUS && node[3][2][0] == NUM && node[3][2][1]->getNumber() == 1 && - !(node[2][0] == "unary-prefix" && node[2][1] == "~")) { // avoid creating ~~~ which is confusing for asm given the role of ~~ - safeCopy(node, make2("unary-prefix", "~", node[2])); + !(node[2][0] == UNARY_PREFIX && node[2][1] == B_NOT)) { // avoid creating ~~~ which is confusing for asm given the role of ~~ + safeCopy(node, make2(UNARY_PREFIX, B_NOT, node[2])); return; } - } else if (type == "binary" && node[1] == ">>" && node[3][0] == "num" && - node[2][0] == "binary" && node[2][1] == "<<" && node[2][3][0] == "num" && - node[2][2][0] == "sub" && node[2][2][1][0] == "name") { + } else if (type == BINARY && node[1] == RSHIFT && node[3][0] == NUM && + node[2][0] == BINARY && node[2][1] == LSHIFT && node[2][3][0] == NUM && + node[2][2][0] == SUB && node[2][2][1][0] == NAME) { // collapse HEAPU?8[..] << 24 >> 24 etc. into HEAP8[..] | 0 double amount = node[3][1]->getNumber(); if (amount == node[2][3][1]->getNumber()) { - HeapInfo hi = parseHeap(node[2][2][1][1]->getString()); + HeapInfo hi = parseHeap(node[2][2][1][1]->getCString()); if (hi.valid && hi.bits == 32 - amount) { node[2][2][1][1]->setString(getHeapStr(hi.bits, false)); - node[1]->setString("|"); + node[1]->setString(OR); node[2] = node[2][2]; node[3][1]->setNumber(0); rerunOrZeroPass = true; return; } } - } else if (type == "assign") { + } else if (type == ASSIGN) { // optimizations for assigning into HEAP32 specifically - if (node[1]->isBool(true) && node[2][0] == "sub" && node[2][1][0] == "name") { - if (node[2][1][1] == "HEAP32") { + if (node[1]->isBool(true) && node[2][0] == SUB && node[2][1][0] == NAME) { + if (node[2][1][1] == HEAP32) { // HEAP32[..] = x | 0 does not need the | 0 (unless it is a mandatory |0 of a call) - if (node[3][0] == "binary" && node[3][1] == "|") { - if (node[3][2][0] == "num" && node[3][2][1]->getNumber() == 0 && node[3][3][0] != "call") { + if (node[3][0] == BINARY && node[3][1] == OR) { + if (node[3][2][0] == NUM && node[3][2][1]->getNumber() == 0 && node[3][3][0] != CALL) { node[3] = node[3][3]; - } else if (node[3][3][0] == "num" && node[3][3][1]->getNumber() == 0 && node[3][2][0] != "call") { + } else if (node[3][3][0] == NUM && node[3][3][1]->getNumber() == 0 && node[3][2][0] != CALL) { node[3] = node[3][2]; } } - } else if (node[2][1][1] == "HEAP8") { + } else if (node[2][1][1] == HEAP8) { // HEAP8[..] = x & 0xff does not need the & 0xff - if (node[3][0] == "binary" && node[3][1] == "&" && node[3][3][0] == "num" && node[3][3][1]->getNumber() == 0xff) { + if (node[3][0] == BINARY && node[3][1] == AND && node[3][3][0] == NUM && node[3][3][1]->getNumber() == 0xff) { node[3] = node[3][2]; } - } else if (node[2][1][1] == "HEAP16") { + } else if (node[2][1][1] == HEAP16) { // HEAP16[..] = x & 0xffff does not need the & 0xffff - if (node[3][0] == "binary" && node[3][1] == "&" && node[3][3][0] == "num" && node[3][3][1]->getNumber() == 0xffff) { + if (node[3][0] == BINARY && node[3][1] == AND && node[3][3][0] == NUM && node[3][3][1]->getNumber() == 0xffff) { node[3] = node[3][2]; } } } Ref value = node[3]; - if (value[0] == "binary" && value[1] == "|") { + if (value[0] == BINARY && value[1] == OR) { // canonicalize order of |0 to end - if (value[2][0] == "num" && value[2][1]->getNumber() == 0) { + if (value[2][0] == NUM && value[2][1]->getNumber() == 0) { Ref temp = value[2]; value[2] = value[3]; value[3] = temp; } // if a seq ends in an |0, remove an external |0 // note that it is only safe to do this in assigns, like we are doing here (return (x, y|0); is not valid) - if (value[2][0] == "seq" && value[2][2][0] == "binary" && USEFUL_BINARY_OPS.has(value[2][2][1])) { + if (value[2][0] == SEQ && value[2][2][0] == BINARY && USEFUL_BINARY_OPS.has(value[2][2][1])) { node[3] = value[2]; } } - } else if (type == "binary" && node[1] == ">>" && node[2][0] == "num" && node[3][0] == "num") { + } else if (type == BINARY && node[1] == RSHIFT && node[2][0] == NUM && node[3][0] == NUM) { // optimize num >> num, in asm we need this since we do not run optimizeShifts - node[0]->setString("num"); + node[0]->setString(NUM); node[1]->setNumber(int(node[2][1]->getNumber()) >> int(node[3][1]->getNumber())); node->setSize(2); return; - } else if (type == "binary" && node[1] == "+") { + } else if (type == BINARY && node[1] == PLUS) { // The most common mathop is addition, e.g. in getelementptr done repeatedly. We can join all of those, // by doing (num+num) ==> newnum, and (name+num)+num = name+newnum - if (node[2][0] == "num" && node[3][0] == "num") { + if (node[2][0] == NUM && node[3][0] == NUM) { node[2][1]->setNumber(int(node[2][1]->getNumber()) + int(node[3][1]->getNumber())); safeCopy(node, node[2]); return; @@ -957,7 +1002,7 @@ void simplifyExpressions(Ref ast) { for (int i = 2; i <= 3; i++) { int ii = 5-i; for (int j = 2; j <= 3; j++) { - if (node[i][0] == "num" && node[ii][0] == "binary" && node[ii][1] == "+" && node[ii][j][0] == "num") { + if (node[i][0] == NUM && node[ii][0] == BINARY && node[ii][1] == PLUS && node[ii][j][0] == NUM) { node[ii][j][1]->setNumber(int(node[ii][j][1]->getNumber()) + int(node[i][1]->getNumber())); safeCopy(node, node[ii]); return; @@ -973,34 +1018,34 @@ void simplifyExpressions(Ref ast) { AsmData asmData(ast); traversePre(ast, [](Ref node) { Ref type = node[0]; - if (type == "assign") { - if (node[1]->isBool(true) && node[2][0] == "sub" && node[2][1][0] == "name" && node[2][1][1] == "HEAP32") { + if (type == ASSIGN) { + if (node[1]->isBool(true) && node[2][0] == SUB && node[2][1][0] == NAME && node[2][1][1] == HEAP32) { // remove bitcasts that are now obviously pointless, e.g. // HEAP32[$45 >> 2] = HEAPF32[tempDoublePtr >> 2] = ($14 < $28 ? $14 : $28) - $42, HEAP32[tempDoublePtr >> 2] | 0; Ref value = node[3]; - if (value[0] == "seq" && value[1][0] == "assign" && value[1][2][0] == "sub" && value[1][2][1][0] == "name" && value[1][2][1][1] == "HEAPF32" && - value[1][2][2][0] == "binary" && value[1][2][2][2][0] == "name" && value[1][2][2][2][1] == "tempDoublePtr") { + if (value[0] == SEQ && value[1][0] == ASSIGN && value[1][2][0] == SUB && value[1][2][1][0] == NAME && value[1][2][1][1] == HEAPF32 && + value[1][2][2][0] == BINARY && value[1][2][2][2][0] == NAME && value[1][2][2][2][1] == TEMP_DOUBLE_PTR) { // transform to HEAPF32[$45 >> 2] = ($14 < $28 ? $14 : $28) - $42; - node[2][1][1]->setString("HEAPF32"); + node[2][1][1]->setString(HEAPF32); node[3] = value[1][3]; } } - } else if (type == "seq") { + } else if (type == SEQ) { // (HEAP32[tempDoublePtr >> 2] = HEAP32[$37 >> 2], +HEAPF32[tempDoublePtr >> 2]) // ==> // +HEAPF32[$37 >> 2] - if (node[0] == "seq" && node[1][0] == "assign" && node[1][2][0] == "sub" && node[1][2][1][0] == "name" && - (node[1][2][1][1] == "HEAP32" || node[1][2][1][1] == "HEAPF32") && - node[1][2][2][0] == "binary" && node[1][2][2][2][0] == "name" && node[1][2][2][2][1] == "tempDoublePtr" && - node[1][3][0] == "sub" && node[1][3][1][0] == "name" && (node[1][3][1][1] == "HEAP32" || node[1][3][1][1] == "HEAPF32") && - node[2][0] != "seq") { // avoid (x, y, z) which can be used for tempDoublePtr on doubles for alignment fixes - if (node[1][2][1][1] == "HEAP32") { - node[1][3][1][1]->setString("HEAPF32"); + if (node[0] == SEQ && node[1][0] == ASSIGN && node[1][2][0] == SUB && node[1][2][1][0] == NAME && + (node[1][2][1][1] == HEAP32 || node[1][2][1][1] == HEAPF32) && + node[1][2][2][0] == BINARY && node[1][2][2][2][0] == NAME && node[1][2][2][2][1] == TEMP_DOUBLE_PTR && + node[1][3][0] == SUB && node[1][3][1][0] == NAME && (node[1][3][1][1] == HEAP32 || node[1][3][1][1] == HEAPF32) && + node[2][0] != SEQ) { // avoid (x, y, z) which can be used for tempDoublePtr on doubles for alignment fixes + if (node[1][2][1][1] == HEAP32) { + node[1][3][1][1]->setString(HEAPF32); safeCopy(node, makeAsmCoercion(node[1][3], detectType(node[2]))); return; } else { - node[1][3][1][1]->setString("HEAP32"); - safeCopy(node, make3("binary", "|", node[1][3], makeNum(0))); + node[1][3][1][1]->setString(HEAP32); + safeCopy(node, make3(BINARY, OR, node[1][3], makeNum(0))); return; } } @@ -1018,17 +1063,17 @@ void simplifyExpressions(Ref ast) { }; std::unordered_map bitcastVars; traversePre(ast, [&bitcastVars](Ref node) { - if (node[0] == "assign" && node[1]->isBool(true) && node[2][0] == "name") { + if (node[0] == ASSIGN && node[1]->isBool(true) && node[2][0] == NAME) { Ref value = node[3]; - if (value[0] == "seq" && value[1][0] == "assign" && value[1][2][0] == "sub" && value[1][2][1][0] == "name" && - (value[1][2][1][1] == "HEAP32" || value[1][2][1][1] == "HEAPF32") && - value[1][2][2][0] == "binary" && value[1][2][2][2][0] == "name" && value[1][2][2][2][1] == "tempDoublePtr") { - std::string name = node[2][1]->getString(); - std::string heap = value[1][2][1][1]->getString(); - if (heap == "HEAP32") { + if (value[0] == SEQ && value[1][0] == ASSIGN && value[1][2][0] == SUB && value[1][2][1][0] == NAME && + (value[1][2][1][1] == HEAP32 || value[1][2][1][1] == HEAPF32) && + value[1][2][2][0] == BINARY && value[1][2][2][2][0] == NAME && value[1][2][2][2][1] == TEMP_DOUBLE_PTR) { + std::string name = node[2][1]->getCString(); + std::string heap = value[1][2][1][1]->getCString(); + if (heap[4] == '3') { // "HEAP32" bitcastVars[name].define_HEAP32++; } else { - assert(heap == "HEAPF32"); + assert(heap[4] == 'F'); // HEAPF32 bitcastVars[name].define_HEAPF32++; } bitcastVars[name].defines.push_back(node); @@ -1038,16 +1083,16 @@ void simplifyExpressions(Ref ast) { }); traversePre(ast, [&bitcastVars](Ref node) { Ref type = node[0]; - if (type == "name" && bitcastVars[node[1]->getString()].ok) { - bitcastVars[node[1]->getString()].namings++; - } else if (type == "assign" && node[1]->isBool(true)) { + if (type == NAME && bitcastVars[node[1]->getCString()].ok) { + bitcastVars[node[1]->getCString()].namings++; + } else if (type == ASSIGN && node[1]->isBool(true)) { Ref value = node[3]; - if (value[0] == "name") { - std::string name = value[1]->getString(); + if (value[0] == NAME) { + std::string name = value[1]->getCString(); if (bitcastVars[name].ok) { Ref target = node[2]; - if (target[0] == "sub" && target[1][0] == "name" && (target[1][1] == "HEAP32" || target[1][1] == "HEAPF32")) { - if (target[1][1] == "HEAP32") { + if (target[0] == SUB && target[1][0] == NAME && (target[1][1] == HEAP32 || target[1][1] == HEAPF32)) { + if (target[1][1] == HEAP32) { bitcastVars[name].use_HEAP32++; } else { bitcastVars[name].use_HEAPF32++; @@ -1066,12 +1111,13 @@ void simplifyExpressions(Ref ast) { info.define_HEAP32+info.define_HEAPF32 > 0 && info.use_HEAP32+info.use_HEAPF32 > 0 && info.define_HEAP32*info.use_HEAP32 == 0 && info.define_HEAPF32*info.use_HEAPF32 == 0 && asmData.isLocal(v) && info.namings == info.define_HEAP32+info.define_HEAPF32+info.use_HEAP32+info.use_HEAPF32) { - std::string correct = info.use_HEAP32 ? "HEAPF32" : "HEAP32"; + std::string correct = (info.use_HEAP32 ? HEAPF32 : HEAP32).c_str(); for (auto define : info.defines) { define[3] = define[3][1][3]; - if (correct == "HEAP32") { - define[3] = make3("binary", "|", define[3], makeNum(0)); + if (correct[4] == '3') { // HEAP32 + define[3] = make3(BINARY, OR, define[3], makeNum(0)); } else { + assert(correct[4] == 'F'); // HEAPF32 define[3] = makeAsmCoercion(define[3], preciseF32 ? ASM_FLOAT : ASM_DOUBLE); } // do we want a simplifybitops on the new values here? @@ -1093,13 +1139,13 @@ void simplifyExpressions(Ref ast) { }; std::function emitsBoolean = [&emitsBoolean](Ref node) { - std::string type = node[0]->getString(); - if (type == "num") { + IString type = node[0]->getIString(); + if (type == NUM) { return node[1]->getNumber() == 0 || node[1]->getNumber() == 1; } - if (type == "binary") return COMPARE_OPS.has(node[1]); - if (type == "unary-prefix") return node[1] == "!"; - if (type == "conditional") return emitsBoolean(node[2]) && emitsBoolean(node[3]); + if (type == BINARY) return COMPARE_OPS.has(node[1]); + if (type == UNARY_PREFIX) return node[1] == L_NOT; + if (type == CONDITIONAL) return emitsBoolean(node[2]) && emitsBoolean(node[3]); return false; }; @@ -1109,7 +1155,7 @@ void simplifyExpressions(Ref ast) { auto conditionalize = [&emitsBoolean](Ref ast) { const int MIN_COST = 7; traversePre(ast, [&emitsBoolean](Ref node) { - if (node[0] == "binary" && (node[1] == "|" || node[1] == "&") && node[3][0] != "num" && node[2][0] != "num") { + if (node[0] == BINARY && (node[1] == OR || node[1] == AND) && node[3][0] != NUM && node[2][0] != NUM) { // logical operator on two non-numerical values Ref left = node[2]; Ref right = node[3]; @@ -1139,12 +1185,12 @@ void simplifyExpressions(Ref ast) { } // worth it, perform conditionalization Ref ret; - if (node[1] == "|") { - ret = make3("conditional", left, makeNum(1), right); + if (node[1] == OR) { + ret = make3(CONDITIONAL, left, makeNum(1), right); } else { // & - ret = make3("conditional", left, right, makeNum(0)); + ret = make3(CONDITIONAL, left, right, makeNum(0)); } - if (left[0] == "unary-prefix" && left[1] == "!") { + if (left[0] == UNARY_PREFIX && left[1] == L_NOT) { ret[1] = flipCondition(left); Ref temp = ret[2]; ret[2] = ret[3]; @@ -1175,20 +1221,20 @@ void simplifyIfs(Ref ast) { traversePre(func, [&simplifiedAnElse](Ref node) { // simplify if (x) { if (y) { .. } } to if (x ? y : 0) { .. } - if (node[0] == "if") { + if (node[0] == IF) { Ref body = node[2]; // recurse to handle chains - while (body[0] == "block") { + while (body[0] == BLOCK) { Ref stats = body[1]; if (stats->size() == 0) break; Ref other = stats[stats->size()-1]; - if (other[0] != "if") { + if (other[0] != IF) { // our if block does not end with an if. perhaps if have an else we can flip - if (!!node[3] && node[3][0] == "block") { + if (!!node[3] && node[3][0] == BLOCK) { stats = node[3][1]; if (stats->size() == 0) break; other = stats[stats->size()-1]; - if (other[0] == "if") { + if (other[0] == IF) { // flip node node[1] = flipCondition(node[1]); node[2] = node[3]; @@ -1222,7 +1268,7 @@ void simplifyIfs(Ref ast) { if (!ok) break; for (int i = stats->size()-2; i >= 0; i--) { Ref curr = deStat(stats[i]); - other[1] = make2("seq", curr, other[1]); + other[1] = make2(SEQ, curr, other[1]); } Ref temp = makeArray(); temp->push_back(other); @@ -1230,7 +1276,7 @@ void simplifyIfs(Ref ast) { } if (stats->size() != 1) break; if (!!node[3]) simplifiedAnElse = true; - node[1] = make3("conditional", node[1], other[1], makeNum(0)); + node[1] = make3(CONDITIONAL, node[1], other[1], makeNum(0)); body = node[2] = other[2]; } } @@ -1247,8 +1293,8 @@ void simplifyIfs(Ref ast) { std::unordered_map labelAssigns; traversePre(func, [&labelAssigns, &abort](Ref node) { - if (node[0] == "assign" && node[2][0] == "name" && node[2][1] == "label") { - if (node[3][0] == "num") { + if (node[0] == ASSIGN && node[2][0] == NAME && node[2][1] == LABEL) { + if (node[3][0] == NUM) { int value = node[3][1]->getNumber(); labelAssigns[value] = labelAssigns[value] + 1; } else { @@ -1262,9 +1308,9 @@ void simplifyIfs(Ref ast) { std::unordered_map labelChecks; traversePre(func, [&labelChecks, &abort](Ref node) { - if (node[0] == "binary" && node[1] == "==" && node[2][0] == "binary" && node[2][1] == "|" && - node[2][2][0] == "name" && node[2][2][1] == "label") { - if (node[3][0] == "num") { + if (node[0] == BINARY && node[1] == EQ && node[2][0] == BINARY && node[2][1] == OR && + node[2][2][0] == NAME && node[2][2][1] == LABEL) { + if (node[3][0] == NUM) { int value = node[3][1]->getNumber(); labelChecks[value] = labelChecks[value] + 1; } else { @@ -1277,33 +1323,33 @@ void simplifyIfs(Ref ast) { int inLoop = 0; // when in a loop, we do not emit label = 0; in the relooper as there is no need traversePrePost(func, [&inLoop, &labelAssigns, &labelChecks](Ref node) { - if (node[0] == "while") inLoop++; + if (node[0] == WHILE) inLoop++; Ref stats = getStatements(node); if (!stats->isNull() && stats->size() > 0) { for (int i = 0; i < stats->size()-1; i++) { Ref pre = stats[i]; Ref post = stats[i+1]; - if (pre[0] == "if" && !!pre[3] && post[0] == "if" && !post[3]) { + if (pre[0] == IF && !!pre[3] && post[0] == IF && !post[3]) { Ref postCond = post[1]; - if (postCond[0] == "binary" && postCond[1] == "==" && - postCond[2][0] == "binary" && postCond[2][1] == "|" && - postCond[2][2][0] == "name" && postCond[2][2][1] == "label" && - postCond[2][3][0] == "num" && postCond[2][3][1]->getNumber() == 0 && - postCond[3][0] == "num") { + if (postCond[0] == BINARY && postCond[1] == EQ && + postCond[2][0] == BINARY && postCond[2][1] == OR && + postCond[2][2][0] == NAME && postCond[2][2][1] == LABEL && + postCond[2][3][0] == NUM && postCond[2][3][1]->getNumber() == 0 && + postCond[3][0] == NUM) { double postValue = postCond[3][1]->getNumber(); Ref preElse = pre[3]; - if (labelAssigns[postValue] == 1 && labelChecks[postValue] == 1 && preElse[0] == "block" && preElse->size() >= 2 && preElse[1]->size() == 1) { + if (labelAssigns[postValue] == 1 && labelChecks[postValue] == 1 && preElse[0] == BLOCK && preElse->size() >= 2 && preElse[1]->size() == 1) { Ref preStat = preElse[1][0]; - if (preStat[0] == "stat" && preStat[1][0] == "assign" && - preStat[1][1]->isBool(true) && preStat[1][2][0] == "name" && preStat[1][2][1] == "label" && - preStat[1][3][0] == "num" && preStat[1][3][1]->getNumber() == postValue) { + if (preStat[0] == STAT && preStat[1][0] == ASSIGN && + preStat[1][1]->isBool(true) && preStat[1][2][0] == NAME && preStat[1][2][1] == LABEL && + preStat[1][3][0] == NUM && preStat[1][3][1]->getNumber() == postValue) { // Conditions match, just need to make sure the post clears label - if (post[2][0] == "block" && post[2]->size() >= 2 && post[2][1]->size() > 0) { + if (post[2][0] == BLOCK && post[2]->size() >= 2 && post[2][1]->size() > 0) { Ref postStat = post[2][1][0]; bool haveClear = - postStat[0] == "stat" && postStat[1][0] == "assign" && - postStat[1][1]->isBool(true) && postStat[1][2][0] == "name" && postStat[1][2][1] == "label" && - postStat[1][3][0] == "num" && postStat[1][3][1]->getNumber() == 0; + postStat[0] == STAT && postStat[1][0] == ASSIGN && + postStat[1][1]->isBool(true) && postStat[1][2][0] == NAME && postStat[1][2][1] == LABEL && + postStat[1][3][0] == NUM && postStat[1][3][1]->getNumber() == 0; if (!inLoop || haveClear) { // Everything lines up, do it pre[3] = post[2]; @@ -1318,7 +1364,7 @@ void simplifyIfs(Ref ast) { } } }, [&inLoop](Ref node) { - if (node[0] == "while") inLoop--; + if (node[0] == WHILE) inLoop--; }); assert(inLoop == 0); } @@ -1330,15 +1376,15 @@ void optimizeFrounds(Ref ast) { // also emit f0 instead of fround(0) (except in returns) bool inReturn = false, currIsReturn = false; traversePrePost(ast, [&](Ref node) { - currIsReturn = node[0] == "return"; + currIsReturn = node[0] == RETURN; if (currIsReturn) inReturn = true; }, [&](Ref node) { if (currIsReturn) inReturn = false; - if (node[0] == "call" && node[1][0] == "name" && node[1][1] == "Math_fround") { + if (node[0] == CALL && node[1][0] == NAME && node[1][1] == MATH_FROUND) { Ref arg = node[2][0]; - if (arg[0] == "num") { - if (!inReturn && arg[1]->getNumber() == 0) *node = *makeName("f0"); - } else if (arg[0] == "call" && arg[1][0] == "name" && arg[1][1] == "Math_fround") { + if (arg[0] == NUM) { + if (!inReturn && arg[1]->getNumber() == 0) *node = *makeName(F0); + } else if (arg[0] == CALL && arg[1][0] == NAME && arg[1][1] == MATH_FROUND) { safeCopy(node, arg); } } From 862f81bd58c3e45454501134020c96633c0f883b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 10 Nov 2014 15:20:31 -0800 Subject: [PATCH 080/161] use IString everywhere --- tools/optimizer/minijson.h | 51 ++++++++++++++++++++++--------- tools/optimizer/optimizer.cpp | 57 +++++++++++++++++------------------ 2 files changed, 64 insertions(+), 44 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index ef05aa8dab06c..c23aa80f43603 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -68,24 +69,28 @@ Arena arena; struct IString { const char *str; - class IStringHash : public std::hash { + static size_t hash_c(const char *str) { // TODO: optimize? + uint64_t ret = 0; + while (*str) { + ret = (ret*6364136223846793005ULL) + *str; + str++; + } + return (size_t)ret; + } + + class CStringHash : public std::hash { public: - size_t operator()(const char *str) const { // TODO: optimize? - uint64_t ret = 0; - while (*str) { - ret = (ret*6364136223846793005ULL) + *str; - str++; - } - return (size_t)ret; + size_t operator()(const char *str) const { + return IString::hash_c(str); } }; - class IStringEqual : public std::equal_to { + class CStringEqual : public std::equal_to { public: - bool operator()(const char * const &x, const char * const &y) const { + bool operator()(const char *x, const char *y) const { return strcmp(x, y) == 0; } }; - typedef std::unordered_set StringSet; + typedef std::unordered_set StringSet; static StringSet strings; IString() : str(nullptr) {} @@ -103,11 +108,11 @@ struct IString { str = s.str; } - bool operator==(const IString& other) { + bool operator==(const IString& other) const { assert((str == other.str) == !strcmp(str, other.str)); return str == other.str; // fast! } - bool operator!=(const IString& other) { + bool operator!=(const IString& other) const { assert((str == other.str) == !strcmp(str, other.str)); return str != other.str; // fast! } @@ -116,13 +121,31 @@ struct IString { return str[x]; } - const char *c_str() { return str; } + const char *c_str() const { return str; } bool isNull() { return str == nullptr; } }; IString::StringSet IString::strings; +// Utilities for creating hashmaps/sets over IStrings + +namespace std { + + template <> struct hash : public unary_function { + size_t operator()(const IString& str) const { + return IString::hash_c(str.c_str()); + } + }; + + template <> struct equal_to : public binary_function { + bool operator()(const IString& x, const IString& y) const { + return x == y; + } + }; + +} + // Main value type struct Value { enum Type { diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 78a6658899402..5cbced20bccbc 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -208,7 +207,7 @@ struct AsmData; AsmType detectType(Ref node, AsmData *asmData=nullptr, bool inVarDef=false); Ref makeEmpty(); bool isEmpty(Ref node); -Ref makeAsmVarDef(const std::string& v_, AsmType type); +Ref makeAsmVarDef(const IString& v, AsmType type); Ref makeArray(); Ref makeNum(double x); Ref makeName(IString str); @@ -221,7 +220,7 @@ struct AsmData { AsmType type; bool param; // false if a var }; - typedef std::unordered_map Locals; + typedef std::unordered_map Locals; Locals locals; std::vector params; // in order @@ -230,16 +229,16 @@ struct AsmData { Ref func; - AsmType getType(const std::string& name) { + AsmType getType(const IString& name) { auto ret = locals.find(name); if (ret != locals.end()) return ret->second.type; return ASM_NONE; } - void setType(const std::string& name, AsmType type) { + void setType(const IString& name, AsmType type) { locals[name].type = type; } - bool isLocal(const std::string& name) { + bool isLocal(const IString& name) { return locals.count(name) > 0; } @@ -256,7 +255,7 @@ struct AsmData { Ref name = node[2][1]; int index = func[2]->indexOf(name); if (index < 0) break; // not an assign into a parameter, but a global - std::string str = name->getCString(); + IString& str = name->getIString(); if (locals.count(str) > 0) break; // already done that param, must be starting function body locals[str] = { detectType(node[3]), true }; params.push_back(locals.find(str)); @@ -269,7 +268,7 @@ struct AsmData { if (node[0] != VAR) break; for (int j = 0; j < node[1]->size(); j++) { Ref v = node[1][j]; - std::string name = v[0]->getCString(); + IString& name = v[0]->getIString(); Ref value = v[1]; if (locals.count(name) == 0) { locals[name] = { detectType(value, nullptr, true), false }; @@ -334,7 +333,7 @@ struct AsmData { // add param coercions int next = 0; for (auto param : func[2]->getArray()) { - std::string str = param->getCString(); + IString str = param->getIString(); stats[next++] = make1(STAT, make3(ASSIGN, &(arena.alloc())->setBool(true), makeName(str.c_str()), makeAsmCoercion(makeName(str.c_str()), locals[str].type))); } if (varDefs->size()) { @@ -559,8 +558,7 @@ Ref make3(IString type, Ref a, Ref b, Ref c) { return ret; } -Ref makeAsmVarDef(const std::string& v_, AsmType type) { - Ref v = makeString(IString(v_.c_str())); +Ref makeAsmVarDef(const IString& v, AsmType type) { Ref val; switch (type) { case ASM_INT: val = makeNum(0); break; @@ -583,7 +581,7 @@ Ref makeAsmVarDef(const std::string& v_, AsmType type) { } default: assert(0); } - return makePair(v, val); + return makePair(&(arena.alloc()->setString(v)), val); } Ref makeAsmCoercion(Ref node, AsmType type) { @@ -731,7 +729,7 @@ bool preciseF32 = false; // Optimization passes //===================== -class StringSet : public std::unordered_set { +class StringSet : public std::unordered_set { public: StringSet(const char *init) { // comma-delimited list int size = strlen(init); @@ -746,11 +744,8 @@ class StringSet : public std::unordered_set { } } - bool has(const char *str) { - return count(str) > 0; - } bool has(Ref node) { - return has(node->getCString()); + return count(node->getIString()) > 0; } }; @@ -1061,19 +1056,19 @@ void simplifyExpressions(Ref ast) { BitcastData() : define_HEAP32(0), define_HEAPF32(0), use_HEAP32(0), use_HEAPF32(0), namings(0), ok(false) {} }; - std::unordered_map bitcastVars; + std::unordered_map bitcastVars; traversePre(ast, [&bitcastVars](Ref node) { if (node[0] == ASSIGN && node[1]->isBool(true) && node[2][0] == NAME) { Ref value = node[3]; if (value[0] == SEQ && value[1][0] == ASSIGN && value[1][2][0] == SUB && value[1][2][1][0] == NAME && (value[1][2][1][1] == HEAP32 || value[1][2][1][1] == HEAPF32) && value[1][2][2][0] == BINARY && value[1][2][2][2][0] == NAME && value[1][2][2][2][1] == TEMP_DOUBLE_PTR) { - std::string name = node[2][1]->getCString(); - std::string heap = value[1][2][1][1]->getCString(); - if (heap[4] == '3') { // "HEAP32" + IString name = node[2][1]->getIString(); + IString heap = value[1][2][1][1]->getIString(); + if (heap == HEAP32) { bitcastVars[name].define_HEAP32++; } else { - assert(heap[4] == 'F'); // HEAPF32 + assert(heap == HEAPF32); bitcastVars[name].define_HEAPF32++; } bitcastVars[name].defines.push_back(node); @@ -1088,7 +1083,7 @@ void simplifyExpressions(Ref ast) { } else if (type == ASSIGN && node[1]->isBool(true)) { Ref value = node[3]; if (value[0] == NAME) { - std::string name = value[1]->getCString(); + IString name = value[1]->getIString(); if (bitcastVars[name].ok) { Ref target = node[2]; if (target[0] == SUB && target[1][0] == NAME && (target[1][1] == HEAP32 || target[1][1] == HEAPF32)) { @@ -1104,20 +1099,20 @@ void simplifyExpressions(Ref ast) { } }); for (auto iter : bitcastVars) { - const std::string& v = iter.first; + const IString& v = iter.first; BitcastData& info = iter.second; // good variables define only one type, use only one type, have definitions and uses, and define as a different type than they use if (info.define_HEAP32*info.define_HEAPF32 == 0 && info.use_HEAP32*info.use_HEAPF32 == 0 && info.define_HEAP32+info.define_HEAPF32 > 0 && info.use_HEAP32+info.use_HEAPF32 > 0 && info.define_HEAP32*info.use_HEAP32 == 0 && info.define_HEAPF32*info.use_HEAPF32 == 0 && - asmData.isLocal(v) && info.namings == info.define_HEAP32+info.define_HEAPF32+info.use_HEAP32+info.use_HEAPF32) { - std::string correct = (info.use_HEAP32 ? HEAPF32 : HEAP32).c_str(); + asmData.isLocal(v.c_str()) && info.namings == info.define_HEAP32+info.define_HEAPF32+info.use_HEAP32+info.use_HEAPF32) { + IString& correct = info.use_HEAP32 ? HEAPF32 : HEAP32; for (auto define : info.defines) { define[3] = define[3][1][3]; - if (correct[4] == '3') { // HEAP32 + if (correct == HEAP32) { define[3] = make3(BINARY, OR, define[3], makeNum(0)); } else { - assert(correct[4] == 'F'); // HEAPF32 + assert(correct == HEAPF32); define[3] = makeAsmCoercion(define[3], preciseF32 ? ASM_FLOAT : ASM_DOUBLE); } // do we want a simplifybitops on the new values here? @@ -1126,12 +1121,12 @@ void simplifyExpressions(Ref ast) { use[2][1][1]->setString(correct.c_str()); } AsmType correctType; - switch(asmData.getType(v)) { + switch(asmData.getType(v.c_str())) { case ASM_INT: correctType = preciseF32 ? ASM_FLOAT : ASM_DOUBLE; break; case ASM_FLOAT: case ASM_DOUBLE: correctType = ASM_INT; break; default: {} // pass } - asmData.setType(v, correctType); + asmData.setType(v.c_str(), correctType); } } asmData.denormalize(); @@ -1395,6 +1390,8 @@ void optimizeFrounds(Ref ast) { // Main //================== +#include // only use this for param checking + int main(int argc, char **argv) { // Read input file char *input = argv[1]; From 65abc918259bd593ac439767432fa6aa8f7fe213 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 10 Nov 2014 15:28:07 -0800 Subject: [PATCH 081/161] remove two super-heavy asserts --- tools/optimizer/minijson.h | 7 +++---- tools/optimizer/optimizer.cpp | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index c23aa80f43603..e7bd7a1ef5363 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -45,7 +45,7 @@ struct Ref { bool operator!=(const char *str); bool operator==(const IString &str); bool operator!=(const IString &str); - bool operator==(double d) { assert(0); } // prevent Ref == number, which is potentially ambiguous; use ->getNumber() == number + bool operator==(double d) { assert(0); return false; } // prevent Ref == number, which is potentially ambiguous; use ->getNumber() == number bool operator==(Ref other); bool operator!(); // check if null, in effect }; @@ -100,7 +100,6 @@ struct IString { void set(const char *s) { auto result = strings.insert(s); // if already present, does nothing - //errv("insert IString? %s : %d (size: %u)\n", s, result.second, strings.size()); str = *(result.first); } @@ -109,11 +108,11 @@ struct IString { } bool operator==(const IString& other) const { - assert((str == other.str) == !strcmp(str, other.str)); + //assert((str == other.str) == !strcmp(str, other.str)); return str == other.str; // fast! } bool operator!=(const IString& other) const { - assert((str == other.str) == !strcmp(str, other.str)); + //assert((str == other.str) == !strcmp(str, other.str)); return str != other.str; // fast! } diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 5cbced20bccbc..bcd3e3811a2f4 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -470,6 +470,7 @@ AsmType detectType(Ref node, AsmData *asmData, bool inVarDef) { } dump("horrible", node); assert(0); + return ASM_NONE; } // Constructions TODO: share common constructions, and assert they remain frozen From f5c48bab27fb4837ca3ebf4643f1406cbd235f81 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 10 Nov 2014 16:45:21 -0800 Subject: [PATCH 082/161] fix bisect_pair on programs not returning 0 --- tools/bisect_pair.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bisect_pair.py b/tools/bisect_pair.py index cdb7628650460..a4772a97d17ec 100644 --- a/tools/bisect_pair.py +++ b/tools/bisect_pair.py @@ -24,7 +24,7 @@ def path_from_root(*pathelems): rightf.close() def run_code(name): - ret = run_js(name, stderr=PIPE, full_output=True) + ret = run_js(name, stderr=PIPE, full_output=True, assert_returncode=None) # fix stack traces ret = filter(lambda line: not line.startswith(' at ') and not name in line, ret.split('\n')) return '\n'.join(ret) From c930b4941f0000dcb8900e691a01685907d5b142 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 10 Nov 2014 16:46:07 -0800 Subject: [PATCH 083/161] fix double to int conversions --- tools/optimizer/optimizer.cpp | 18 +++++++++++------- tools/test-js-optimizer-asm-pre-output.js | 3 +++ tools/test-js-optimizer-asm-pre.js | 5 ++++- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index bcd3e3811a2f4..dd497125dea3f 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -70,6 +70,10 @@ IString TOPLEVEL("toplevel"), // Infrastructure //================== +int jsD2I(double x) { + return (int)((int64_t)x); +} + int parseInt(const char *str) { int ret = *str - '0'; while (*(++str)) { @@ -780,7 +784,7 @@ void simplifyExpressions(Ref ast) { double shifts = node[3][1]->getNumber(); if (innerNode[0] == BINARY && innerNode[1] == AND && innerNode[3][0] == NUM) { double mask = innerNode[3][1]->getNumber(); - if (isInteger32(mask) && isInteger32(shifts) && ((int(mask) << int(shifts)) >> int(shifts)) == int(mask)) { + if (isInteger32(mask) && isInteger32(shifts) && ((jsD2I(mask) << jsD2I(shifts)) >> jsD2I(shifts)) == jsD2I(mask)) { safeCopy(node, innerNode); return; } @@ -818,7 +822,7 @@ void simplifyExpressions(Ref ast) { Ref type = node[0]; if (type == BINARY && node[1] == OR) { if (node[2][0] == NUM && node[3][0] == NUM) { - node[2][1]->setNumber(int(node[2][1]->getNumber()) | int(node[3][1]->getNumber())); + node[2][1]->setNumber(jsD2I(node[2][1]->getNumber()) | jsD2I(node[3][1]->getNumber())); stack.push_back(0); safeCopy(node, node[2]); return; @@ -895,14 +899,14 @@ void simplifyExpressions(Ref ast) { if (node[1] == TEMP_DOUBLE_PTR) hasTempDoublePtr = true; } else if (type == BINARY && node[1] == AND && node[3][0] == NUM) { if (node[2][0] == NUM) { - safeCopy(node, makeNum(int(node[2][1]->getNumber()) & int(node[3][1]->getNumber()))); + safeCopy(node, makeNum(jsD2I(node[2][1]->getNumber()) & jsD2I(node[3][1]->getNumber()))); return; } Ref input = node[2]; double amount = node[3][1]->getNumber(); if (input[0] == BINARY && input[1] == AND && input[3][0] == NUM) { // Collapse X & 255 & 1 - node[3][1]->setNumber(int(amount) & int(input[3][1]->getNumber())); + node[3][1]->setNumber(jsD2I(amount) & jsD2I(input[3][1]->getNumber())); node[2] = input[2]; } else if (input[0] == SUB && input[1][0] == NAME) { // HEAP8[..] & 255 => HEAPU8[..] @@ -984,14 +988,14 @@ void simplifyExpressions(Ref ast) { } else if (type == BINARY && node[1] == RSHIFT && node[2][0] == NUM && node[3][0] == NUM) { // optimize num >> num, in asm we need this since we do not run optimizeShifts node[0]->setString(NUM); - node[1]->setNumber(int(node[2][1]->getNumber()) >> int(node[3][1]->getNumber())); + node[1]->setNumber(jsD2I(node[2][1]->getNumber()) >> jsD2I(node[3][1]->getNumber())); node->setSize(2); return; } else if (type == BINARY && node[1] == PLUS) { // The most common mathop is addition, e.g. in getelementptr done repeatedly. We can join all of those, // by doing (num+num) ==> newnum, and (name+num)+num = name+newnum if (node[2][0] == NUM && node[3][0] == NUM) { - node[2][1]->setNumber(int(node[2][1]->getNumber()) + int(node[3][1]->getNumber())); + node[2][1]->setNumber(jsD2I(node[2][1]->getNumber()) + jsD2I(node[3][1]->getNumber())); safeCopy(node, node[2]); return; } @@ -999,7 +1003,7 @@ void simplifyExpressions(Ref ast) { int ii = 5-i; for (int j = 2; j <= 3; j++) { if (node[i][0] == NUM && node[ii][0] == BINARY && node[ii][1] == PLUS && node[ii][j][0] == NUM) { - node[ii][j][1]->setNumber(int(node[ii][j][1]->getNumber()) + int(node[i][1]->getNumber())); + node[ii][j][1]->setNumber(jsD2I(node[ii][j][1]->getNumber()) + jsD2I(node[i][1]->getNumber())); safeCopy(node, node[ii]); return; } diff --git a/tools/test-js-optimizer-asm-pre-output.js b/tools/test-js-optimizer-asm-pre-output.js index da3ebb4f5303e..e4a324c8198a8 100644 --- a/tools/test-js-optimizer-asm-pre-output.js +++ b/tools/test-js-optimizer-asm-pre-output.js @@ -598,4 +598,7 @@ function conditionalizeMe() { print(((HEAP8[a] + HEAP8[b] + HEAP8[c] + HEAP8[d] + HEAP8[e] + HEAP8[f] | 0) > a % b % c % d ? -1 : 1) | $cheap > 0); return (((((Math_imul(i6 + 1, i7) | 0) + 17 | 0) % 5 | 0) == 0 ? 1 : ((((Math_imul(i7 + 1, i7) | 0) + 11 | 0) >>> 0) % 3 | 0) == 0) | 0) == 0; } +function bignum() { + HEAP32[20] = -1515870811; +} diff --git a/tools/test-js-optimizer-asm-pre.js b/tools/test-js-optimizer-asm-pre.js index 08d8c2d43d5b4..9db810f559098 100644 --- a/tools/test-js-optimizer-asm-pre.js +++ b/tools/test-js-optimizer-asm-pre.js @@ -609,4 +609,7 @@ function conditionalizeMe() { print( (((HEAP8[a] + HEAP8[b] + HEAP8[c] + HEAP8[d] + HEAP8[e] + HEAP8[f] | 0) > (a % b % c % d)) ? -1 : 1) | ($cheap > 0) ); return ((((Math_imul(i6+1, i7) | 0) + 17 | 0) % 5 | 0 | 0) == 0 | ((((Math_imul(i7+1, i7) | 0) + 11 | 0) >>> 0) % 3 | 0 | 0) == 0 | 0) == 0; } -// EMSCRIPTEN_GENERATED_FUNCTIONS: ["a", "b", "rett", "ret2t", "retf", "i32_8", "tempDoublePtr", "boxx", "_main", "badf", "badf2", "fcomp", "conditionalizeMe"] +function bignum() { + HEAP32[20] = 2779096485 | 0; +} +// EMSCRIPTEN_GENERATED_FUNCTIONS: ["a", "b", "rett", "ret2t", "retf", "i32_8", "tempDoublePtr", "boxx", "_main", "badf", "badf2", "fcomp", "conditionalizeMe", "bignum"] From 04eddec1444c10f101440a79c50eaf50f8fbad05 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 10 Nov 2014 16:57:29 -0800 Subject: [PATCH 084/161] set thisProgram even if no second param in argv --- src/shell.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/shell.js b/src/shell.js index b1a705c70dc6f..e30c90f35da34 100644 --- a/src/shell.js +++ b/src/shell.js @@ -70,8 +70,12 @@ if (ENVIRONMENT_IS_NODE) { globalEval(read(f)); }; - if (process['argv'].length > 1) + if (process['argv'].length > 1) { Module['thisProgram'] = process['argv'][1].replace(/\\/g, '/'); + } else { + Module['thisProgram'] = 'unknown-program'; + } + Module['arguments'] = process['argv'].slice(2); if (typeof module !== 'undefined') { From 8389f79e7bd9afd63027f32148b0d0e6916ec8ea Mon Sep 17 00:00:00 2001 From: Petr Babicka Date: Tue, 11 Nov 2014 02:04:54 +0100 Subject: [PATCH 085/161] Tests for glfw3 events --- tests/glfw3_events.c | 127 ++++++++++++++++++++++++++++++++++++++++++ tests/test_browser.py | 3 + 2 files changed, 130 insertions(+) create mode 100644 tests/glfw3_events.c diff --git a/tests/glfw3_events.c b/tests/glfw3_events.c new file mode 100644 index 0000000000000..e813b88ce9abb --- /dev/null +++ b/tests/glfw3_events.c @@ -0,0 +1,127 @@ +#include +#include +#include + +#define MULTILINE(...) #__VA_ARGS__ +#define WIDTH 640 +#define HEIGHT 480 + +// Setup tests +typedef struct { + double x, y; + int button; + int action; + int modify; +} test_args_t; + +typedef struct { + char cmd[80]; + test_args_t args; +} test_t; + +// Javascript event.button 0 = left, 1 = middle, 2 = right +test_t g_tests[] = { + { "Module.injectMouseEvent(10.0, 10.0, 'mousedown', 0)", { 10.0, 10.0, GLFW_MOUSE_BUTTON_1, GLFW_PRESS, -1 } }, + { "Module.injectMouseEvent(10.0, 20.0, 'mouseup', 0)", { 10.0, 20.0, GLFW_MOUSE_BUTTON_1, GLFW_RELEASE, -1 } }, + { "Module.injectMouseEvent(10.0, 30.0, 'mousedown', 2)", { 10.0, 30.0, GLFW_MOUSE_BUTTON_2, GLFW_PRESS, -1 } }, + { "Module.injectMouseEvent(10.0, 40.0, 'mouseup', 2)", { 10.0, 40.0, GLFW_MOUSE_BUTTON_2, GLFW_RELEASE, -1 } }, + { "Module.injectMouseEvent(10.0, 50.0, 'mousewheel', 0)", { 10.0, 50.0, -1, -1, -1 } }, + { "Module.injectMouseEvent(10.0, 60.0, 'mousemove', 0)", { 10.0, 60.0, -1, -1, -1 } } +}; + +static unsigned int g_test_actual = 0; +static unsigned int g_test_count = sizeof(g_tests) / sizeof(test_t); +static unsigned int g_state = 0; + +static void on_mouse_callback(GLFWwindow* window, int button, int action, int modify) +{ + test_args_t args = g_tests[g_test_actual].args; + if (args.button == button && args.action == action) + { + g_state |= 1 << g_test_actual; + } + else + { + printf("Test %d: FAIL\n", g_test_actual); + } +} + +static void on_mouse_move(GLFWwindow* window, double x, double y) +{ + test_args_t args = g_tests[g_test_actual].args; + if (args.x == x && args.y == y) + { + g_state |= 1 << g_test_actual; + } + else + { + printf("Test %d: FAIL\n", g_test_actual); + } +} + +static void on_mouse_wheel(GLFWwindow* window, double x, double y) +{ + test_args_t args = g_tests[g_test_actual].args; + if (args.x == x && args.y == y) + { + g_state |= 1 << g_test_actual; + } + else + { + printf("Test %d: FAIL\n", g_test_actual); + } +} + +static void on_error(int error, const char *msg) +{ + printf("%d: %s\n", error, msg); +} + +int main() +{ + GLFWwindow * _mainWindow = NULL; + int result = 0; + unsigned int success = 0; + + emscripten_run_script(MULTILINE( + Module.injectMouseEvent = function(x, y, event_, button) { + var event = document.createEvent("MouseEvents"); + var canvas = Module['canvas']; + event.initMouseEvent(event_, true, true, window, 0, canvas.offsetLeft + x, canvas.offsetTop + y, canvas.offsetLeft + x, canvas.offsetTop + y, 0, 0, 0, 0, button, null); + canvas.dispatchEvent(event); + } + )); + + glfwSetErrorCallback(on_error); + glfwInit(); + printf("%s\n", glfwGetVersionString()); + + glfwWindowHint(GLFW_CLIENT_API, GLFW_OPENGL_ES_API); + _mainWindow = glfwCreateWindow(WIDTH, HEIGHT, "glfw3_events", NULL, NULL); + glfwMakeContextCurrent(_mainWindow); + + glfwSetMouseButtonCallback(_mainWindow, on_mouse_callback); + glfwSetCursorPosCallback(_mainWindow, on_mouse_move); + glfwSetScrollCallback(_mainWindow, on_mouse_wheel); + //glfwSetCharCallback(_mainWindow, ...); + //glfwSetKeyCallback(_mainWindow, ...); + + for (int i = 0; i < g_test_count; ++i) + { + g_test_actual = i; + emscripten_run_script(g_tests[g_test_actual].cmd); + } + + glfwTerminate(); + + success = (1 << (sizeof(g_tests) / sizeof(test_t))) - 1; // (2^count)-1 + +#ifdef REPORT_RESULT + result = g_state == success; + REPORT_RESULT(); +#elif + printf("%d == %d = %d", g_state, success, g_state == success); +#endif + + return 0; +} \ No newline at end of file diff --git a/tests/test_browser.py b/tests/test_browser.py index 51e0e41d80f82..99f7b952076ee 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -2046,6 +2046,9 @@ def test_locate_file(self): def test_glfw3(self): self.btest(path_from_root('tests', 'glfw3.c'), args=['-s', 'LEGACY_GL_EMULATION=1', '-s', 'USE_GLFW=3'], expected='1') + def test_glfw3_events(self): + self.btest(path_from_root('tests', 'glfw3_events.c'), args=['-s', 'LEGACY_GL_EMULATION=1', '-s', 'USE_GLFW=3'], expected='1') + def test_asm_swapping(self): self.clear() open('run.js', 'w').write(r''' From 7e7894b824acd839c9a75f309f3b3a06d72d875d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 10 Nov 2014 17:06:18 -0800 Subject: [PATCH 086/161] disable test_simd8 in emterpreter --- tests/test_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_core.py b/tests/test_core.py index d4693cb37575b..421f11d6adb4a 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -5238,6 +5238,7 @@ def test_simd8(self): Settings.PRECISE_F32 = 1 # SIMD currently requires Math.fround if Settings.ASM_JS: Settings.ASM_JS = 2 # does not validate if os.environ.get('EMCC_FAST_COMPILER') == '0': return self.skip('needs fastcomp') + if self.is_emterpreter(): return self.skip('todo') test_path = path_from_root('tests', 'core', 'test_simd8') src, output = (test_path + s for s in ('.in', '.out')) From edc96afdc4f6fedfc8943ce2bce28acc63bd427d Mon Sep 17 00:00:00 2001 From: Aidan Hobson Sayers Date: Sun, 9 Nov 2014 15:52:59 +0000 Subject: [PATCH 087/161] Only support spidermonkey engine set in ~/.emscripten --- tools/validate_asmjs.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/tools/validate_asmjs.py b/tools/validate_asmjs.py index 26bed51205b12..3fc08aa277112 100644 --- a/tools/validate_asmjs.py +++ b/tools/validate_asmjs.py @@ -14,18 +14,9 @@ import subprocess, sys, re, tempfile, os, time import shared -# Looks up SpiderMonkey engine using the variable SPIDERMONKEY_ENGINE in ~/.emscripten, and if not set up there, via PATH. -def find_spidermonkey_engine(): - sm_engine = shared.SPIDERMONKEY_ENGINE if hasattr(shared, 'SPIDERMONKEY_ENGINE') else [''] - if not sm_engine or len(sm_engine[0]) == 0 or not os.path.exists(sm_engine[0]): - sm_engine[0] = shared.Building.which('js') - if sm_engine[0] == None: - return ['js-not-found'] - return sm_engine - # Given a .js file, returns True/False depending on if that file is valid asm.js def validate_asmjs_jsfile(filename, muteOutput): - cmd = find_spidermonkey_engine() + ['-c', filename] + cmd = shared.SPIDERMONKEY_ENGINE + ['-c', filename] if cmd[0] == 'js-not-found': print >> sys.stderr, 'Could not find SpiderMonkey engine! Please set tis location to SPIDERMONKEY_ENGINE in your ~/.emscripten configuration file!' return False From 91dc09474d21d8dbcd7fe77d24f5d35b6c7eae05 Mon Sep 17 00:00:00 2001 From: Petr Babicka Date: Tue, 11 Nov 2014 02:16:23 +0100 Subject: [PATCH 088/161] Mistype --- tests/glfw3_events.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/glfw3_events.c b/tests/glfw3_events.c index e813b88ce9abb..dec8b20cc2889 100644 --- a/tests/glfw3_events.c +++ b/tests/glfw3_events.c @@ -119,7 +119,7 @@ int main() #ifdef REPORT_RESULT result = g_state == success; REPORT_RESULT(); -#elif +#else printf("%d == %d = %d", g_state, success, g_state == success); #endif From 7827f7c782f2b7e3d667d3a83cc700232164b755 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 10 Nov 2014 17:34:35 -0800 Subject: [PATCH 089/161] fix simplifyNotComps --- tools/optimizer/optimizer.cpp | 4 ++-- tools/test-js-optimizer-asm-pre-output.js | 1 + tools/test-js-optimizer-asm-pre.js | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index dd497125dea3f..39ada9e6e6e01 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -671,11 +671,11 @@ Ref simplifyNotCompsDirect(Ref node) { switch(op->getCString()[0]) { case '<': { if (op == LT) { op->setString(GE); break; } - if (op == LE) { op->setString(GE); break; } + if (op == LE) { op->setString(GT); break; } return node; } case '>': { - if (op == GE) { op->setString(LE); break; } + if (op == GT) { op->setString(LE); break; } if (op == GE) { op->setString(LT); break; } return node; } diff --git a/tools/test-js-optimizer-asm-pre-output.js b/tools/test-js-optimizer-asm-pre-output.js index e4a324c8198a8..f0fcd43b2ae5c 100644 --- a/tools/test-js-optimizer-asm-pre-output.js +++ b/tools/test-js-optimizer-asm-pre-output.js @@ -600,5 +600,6 @@ function conditionalizeMe() { } function bignum() { HEAP32[20] = -1515870811; + if (($2814 | 0) < 0) return; } diff --git a/tools/test-js-optimizer-asm-pre.js b/tools/test-js-optimizer-asm-pre.js index 9db810f559098..f93b1fbdc39d7 100644 --- a/tools/test-js-optimizer-asm-pre.js +++ b/tools/test-js-optimizer-asm-pre.js @@ -611,5 +611,6 @@ function conditionalizeMe() { } function bignum() { HEAP32[20] = 2779096485 | 0; + if (!(($2814 | 0) >= 0)) return; } // EMSCRIPTEN_GENERATED_FUNCTIONS: ["a", "b", "rett", "ret2t", "retf", "i32_8", "tempDoublePtr", "boxx", "_main", "badf", "badf2", "fcomp", "conditionalizeMe", "bignum"] From 31df184c45603c5ba3999af41576a59b993374a4 Mon Sep 17 00:00:00 2001 From: Aidan Hobson Sayers Date: Sun, 9 Nov 2014 12:58:46 +0000 Subject: [PATCH 090/161] Centralise location of llvm source --- tests/test_other.py | 3 +-- tools/shared.py | 46 ++++++++++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/tests/test_other.py b/tests/test_other.py index 7733b5bbed7a9..4c575b219368a 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -2750,8 +2750,7 @@ def test_incorrect_static_call(self): assert ('''unexpected argument type float at index 1 in call to 'doit', should be i32''' in stderr) == asserts, stderr def test_llvm_lit(self): - llvm_src = LLVM_ROOT - while not os.path.exists(os.path.join(llvm_src, 'emscripten-version.txt')): llvm_src = os.path.dirname(llvm_src) + llvm_src = get_fastcomp_src_dir() cmd = [os.path.join(LLVM_ROOT, 'llvm-lit'), '-v', os.path.join(llvm_src, 'test', 'CodeGen', 'JS')] print cmd p = Popen(cmd) diff --git a/tools/shared.py b/tools/shared.py index 9ec88c19372be..a0cf71389bcf0 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -309,6 +309,17 @@ def check_llvm_version(): except Exception, e: logging.warning('Could not verify LLVM version: %s' % str(e)) +# look for emscripten-version.txt files under or alongside the llvm source dir +def get_fastcomp_src_dir(): + d = LLVM_ROOT + while d != os.path.dirname(d): + # look for version file in llvm repo, making sure not to mistake the emscripten repo for it + if os.path.exists(os.path.join(d, 'emscripten-version.txt')) and not os.path.abspath(d) == os.path.abspath(path_from_root()): + return d + else: + d = os.path.dirname(d) + return None + def check_fastcomp(): try: llc_version_info = Popen([LLVM_COMPILER, '--version'], stdout=PIPE).communicate()[0] @@ -321,27 +332,20 @@ def check_fastcomp(): logging.critical('you can fall back to the older (pre-fastcomp) compiler core, although that is not recommended, see https://github.com/kripken/emscripten/wiki/LLVM-Backend') return False - # look for a source tree under the llvm binary directory. if there is one, look for emscripten-version.txt files - seen = False - d = os.path.dirname(LLVM_COMPILER) - while d != os.path.dirname(d): - # look for version file in llvm repo, making sure not to mistake the emscripten repo for it - if os.path.exists(os.path.join(d, 'emscripten-version.txt')) and not os.path.abspath(d) == os.path.abspath(path_from_root()): - seen = True - llvm_version = open(os.path.join(d, 'emscripten-version.txt')).read().strip() - if os.path.exists(os.path.join(d, 'tools', 'clang', 'emscripten-version.txt')): - clang_version = open(os.path.join(d, 'tools', 'clang', 'emscripten-version.txt')).read().strip() - elif os.path.exists(os.path.join(d, 'tools', 'clang')): - clang_version = '?' # Looks like the LLVM compiler tree has an old checkout from the time before it contained a version.txt: Should update! - else: - clang_version = llvm_version # This LLVM compiler tree does not have a tools/clang, so it's probably an out-of-source build directory. No need for separate versioning. - if EMSCRIPTEN_VERSION != llvm_version or EMSCRIPTEN_VERSION != clang_version: - logging.error('Emscripten, llvm and clang versions do not match, this is dangerous (%s, %s, %s)', EMSCRIPTEN_VERSION, llvm_version, clang_version) - logging.error('Make sure to use the same branch in each repo, and to be up-to-date on each. See https://github.com/kripken/emscripten/wiki/LLVM-Backend') - break - d = os.path.dirname(d) - if not seen: - logging.warning('did not see a source tree above the LLVM root directory (guessing based on directory of %s), could not verify version numbers match' % LLVM_COMPILER) + d = get_fastcomp_src_dir() + if d is not None: + llvm_version = open(os.path.join(d, 'emscripten-version.txt')).read().strip() + if os.path.exists(os.path.join(d, 'tools', 'clang', 'emscripten-version.txt')): + clang_version = open(os.path.join(d, 'tools', 'clang', 'emscripten-version.txt')).read().strip() + elif os.path.exists(os.path.join(d, 'tools', 'clang')): + clang_version = '?' # Looks like the LLVM compiler tree has an old checkout from the time before it contained a version.txt: Should update! + else: + clang_version = llvm_version # This LLVM compiler tree does not have a tools/clang, so it's probably an out-of-source build directory. No need for separate versioning. + if EMSCRIPTEN_VERSION != llvm_version or EMSCRIPTEN_VERSION != clang_version: + logging.error('Emscripten, llvm and clang versions do not match, this is dangerous (%s, %s, %s)', EMSCRIPTEN_VERSION, llvm_version, clang_version) + logging.error('Make sure to use the same branch in each repo, and to be up-to-date on each. See https://github.com/kripken/emscripten/wiki/LLVM-Backend') + else: + logging.warning('did not see a source tree above or next to the LLVM root directory (guessing based on directory of %s), could not verify version numbers match' % LLVM_COMPILER) return True except Exception, e: logging.warning('could not check fastcomp: %s' % str(e)) From 82e068c507415a8dfb19217ee840548c52f4fe67 Mon Sep 17 00:00:00 2001 From: Aidan Hobson Sayers Date: Sun, 9 Nov 2014 13:20:53 +0000 Subject: [PATCH 091/161] Fix finding llvm src dir for current emsdk layout --- tools/shared.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index a0cf71389bcf0..bcc1ece9ac78f 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -312,10 +312,16 @@ def check_llvm_version(): # look for emscripten-version.txt files under or alongside the llvm source dir def get_fastcomp_src_dir(): d = LLVM_ROOT + emroot = path_from_root() # already abspath + # look for version file in llvm repo, making sure not to mistake the emscripten repo for it while d != os.path.dirname(d): - # look for version file in llvm repo, making sure not to mistake the emscripten repo for it - if os.path.exists(os.path.join(d, 'emscripten-version.txt')) and not os.path.abspath(d) == os.path.abspath(path_from_root()): + d = os.path.abspath(d) + # when the build directory lives below the source directory + if os.path.exists(os.path.join(d, 'emscripten-version.txt')) and not d == emroot: return d + # when the build directory lives alongside the source directory + elif os.path.exists(os.path.join(d, 'src', 'emscripten-version.txt')) and not os.path.join(d, 'src') == emroot: + return os.path.join(d, 'src') else: d = os.path.dirname(d) return None From 4787305c9ac24c3da639b0bbe75b2b2562d5352d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 10 Nov 2014 17:49:12 -0800 Subject: [PATCH 092/161] update sanity.test_llvm_fastcomp --- tests/test_sanity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sanity.py b/tests/test_sanity.py index d1739e9445c13..af93077006b4f 100644 --- a/tests/test_sanity.py +++ b/tests/test_sanity.py @@ -238,7 +238,7 @@ def test_llvm_fastcomp(self): os.chmod(path_from_root('tests', 'fake', 'bin', 'llc'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) os.chmod(path_from_root('tests', 'fake', 'bin', 'clang++'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) try_delete(SANITY_FILE) - output = self.check_working(EMCC, 'did not see a source tree above the LLVM root directory') + output = self.check_working(EMCC, 'did not see a source tree above or next to the LLVM root directory') VERSION_WARNING = 'Emscripten, llvm and clang versions do not match, this is dangerous' From 220b8b33018b0dfbe7e63b5a56385e299f68a5cf Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 10 Nov 2014 17:50:56 -0800 Subject: [PATCH 093/161] do not assume all JS engine variables exist --- tools/shared.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index bcc1ece9ac78f..45fb9c9f04fd6 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -263,9 +263,19 @@ def listify(x): if type(x) is not list: return [x] return x -SPIDERMONKEY_ENGINE = listify(SPIDERMONKEY_ENGINE) -NODE_JS = listify(NODE_JS) -V8_ENGINE = listify(V8_ENGINE) +try: + SPIDERMONKEY_ENGINE = listify(SPIDERMONKEY_ENGINE) +except: + pass +try: + NODE_JS = listify(NODE_JS) +except: + pass +try: + V8_ENGINE = listify(V8_ENGINE) +except: + pass + COMPILER_ENGINE = listify(COMPILER_ENGINE) JS_ENGINES = [listify(ENGINE) for ENGINE in JS_ENGINES] From 21e3fe78a379e6451984f856d3454e7dc0bb5674 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Nov 2014 10:06:37 -0800 Subject: [PATCH 094/161] fix engine banning in test runner to only look at first parameter of listified engines --- tests/runner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/runner.py b/tests/runner.py index 9dc5222a20c1d..e223a7300dc9b 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -468,7 +468,9 @@ def do_run(self, src, expected_output, args=[], output_nicerizer=None, output_pr # Run in both JavaScript engines, if optimizing - significant differences there (typed arrays) if js_engines is None: js_engines = JS_ENGINES - js_engines = filter(lambda engine: engine not in self.banned_js_engines, js_engines) + for engine in js_engines: assert type(engine) == list + for engine in self.banned_js_engines: assert type(engine) == list + js_engines = filter(lambda engine: engine[0] not in map(lambda engine: engine[0], self.banned_js_engines), js_engines) if len(js_engines) == 0: return self.skip('No JS engine present to run this test with. Check %s and the paths therein.' % EM_CONFIG) for engine in js_engines: js_output = self.run_generated_code(engine, filename + '.o.js', args, output_nicerizer=output_nicerizer, assert_returncode=assert_returncode) From 767b573f9f2a34999bf41f82cacb64445e789432 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Nov 2014 13:33:28 -0800 Subject: [PATCH 095/161] asmData.deleteVar --- tools/optimizer/optimizer.cpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 39ada9e6e6e01..fc718431a86a3 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -227,8 +227,8 @@ struct AsmData { typedef std::unordered_map Locals; Locals locals; - std::vector params; // in order - std::vector vars; // in order + std::vector params; // in order + std::vector vars; // in order AsmType ret; Ref func; @@ -262,7 +262,7 @@ struct AsmData { IString& str = name->getIString(); if (locals.count(str) > 0) break; // already done that param, must be starting function body locals[str] = { detectType(node[3]), true }; - params.push_back(locals.find(str)); + params.push_back(str); stats[i] = makeEmpty(); i++; } @@ -276,7 +276,7 @@ struct AsmData { Ref value = v[1]; if (locals.count(name) == 0) { locals[name] = { detectType(value, nullptr, true), false }; - vars.push_back(locals.find(name)); + vars.push_back(name); v->setSize(1); // make an un-assigning var } else { assert(j == 0); // cannot break in the middle @@ -319,7 +319,7 @@ struct AsmData { // calculate variable definitions Ref varDefs = makeArray(); for (auto v : vars) { - varDefs->push_back(makeAsmVarDef(v->first, v->second.type)); + varDefs->push_back(makeAsmVarDef(v, locals[v].type)); } // each param needs a line; reuse emptyNodes as much as we can int numParams = params.size(); @@ -367,6 +367,15 @@ struct AsmData { //printErr('denormalized \n\n' + astToSrc(func) + '\n\n'); } + void deleteVar(IString name) { + locals.erase(name); + for (int i = 0; i < vars.size(); i++) { + if (vars[i] == name) { + vars.erase(vars.begin() + i); + break; + } + } + } }; struct HeapInfo { From be68d22ae9cb4d47e62faa4ec07dd52f35f97fec Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Nov 2014 14:17:33 -0800 Subject: [PATCH 096/161] fix missing var in js optimizer --- tools/js-optimizer.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/js-optimizer.js b/tools/js-optimizer.js index 53a5ec583b709..f592c7eefe091 100644 --- a/tools/js-optimizer.js +++ b/tools/js-optimizer.js @@ -4403,6 +4403,7 @@ function eliminate(ast, memSafe) { var info = tracked[name]; delete tracked[name]; var defNode = info.defNode; + var value; if (!sideEffectFree[name]) { if (defNode[0] === 'var') { defNode[1].forEach(function(pair) { From e5de2cd5e552aa95b7a9214d31bb8e4cd8e83a5a Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Nov 2014 14:17:58 -0800 Subject: [PATCH 097/161] initial porting of eliminator to native optimizer --- tools/optimizer/optimizer.cpp | 768 +++++++++++++++++++++++++++++++++- 1 file changed, 764 insertions(+), 4 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index fc718431a86a3..8e1d373000d57 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -24,11 +24,13 @@ IString TOPLEVEL("toplevel"), RETURN("return"), IF("if"), WHILE("while"), + DO("do"), SEQ("seq"), SUB("sub"), CALL("call"), NUM("num"), LABEL("label"), + SWITCH("switch"), STRING("string"), INF("inf"), NaN("nan"), @@ -64,7 +66,8 @@ IString TOPLEVEL("toplevel"), HEAPU16("HEAPU16"), HEAPU32("HEAPU32"), HEAPF64("HEAPF64"), - F0("f0"); + F0("f0"), + EMPTY(""); //================== // Infrastructure @@ -715,6 +718,32 @@ void safeCopy(Ref target, Ref source) { // safely copy source onto target, even *target = *temp; } +void removeAllEmptySubNodes(Ref ast) { + traversePre(ast, [](Ref node) { + int index = -1; + if (node[0] == DEFUN) { + index = 3; + } else if (node[0] == BLOCK && !!node[1]) { + index = 1; + } + if (index > 0) { + Ref arr = node[index]; + int skip = 0; + for (int i = 0; i < arr->size(); i++) { + if (skip) { + arr[i-skip] = arr[i]; + } + if (isEmpty(deStat(arr[i]))) { + skip++; + } + } + if (skip) arr->setSize(arr->size() - skip); + } else if (node[0] == SEQ && isEmpty(node[1])) { + safeCopy(node, node[2]); + } + }); +} + // Calculations int measureCost(Ref ast) { @@ -743,8 +772,17 @@ bool preciseF32 = false; // Optimization passes //===================== +#define HASES \ + bool has(const IString& str) { \ + return count(str) > 0; \ + } \ + bool has(Ref node) { \ + return count(node->getIString()) > 0; \ + } + class StringSet : public std::unordered_set { public: + StringSet() {} StringSet(const char *init) { // comma-delimited list int size = strlen(init); char *curr = (char*)malloc(size+1); // leaked! @@ -758,9 +796,7 @@ class StringSet : public std::unordered_set { } } - bool has(Ref node) { - return count(node->getIString()) > 0; - } + HASES }; StringSet USEFUL_BINARY_OPS("<< >> | & ^"), @@ -770,6 +806,9 @@ StringSet USEFUL_BINARY_OPS("<< >> | & ^"), COERCION_REQUIRING_OPS("sub unary-prefix"), // ops that in asm must be coerced right away COERCION_REQUIRING_BINARIES("* / %"); // binary ops that in asm must be coerced +StringSet ASSOCIATIVE_BINARIES("+ * | & ^"), + NAME_OR_NUM("name num"); + bool isFunctionTable(const char *name) { static const char *functionTable = "FUNCTION_TABLE"; static unsigned size = strlen(functionTable); @@ -780,6 +819,725 @@ bool isFunctionTable(Ref value) { return value->isString() && isFunctionTable(value->getCString()); } +// Passes + +// Eliminator aka Expressionizer +// +// The goal of this pass is to eliminate unneeded variables (which represent one of the infinite registers in the LLVM +// model) and thus to generate complex expressions where possible, for example +// +// var x = a(10); +// var y = HEAP[20]; +// print(x+y); +// +// can be transformed into +// +// print(a(10)+HEAP[20]); +// +// The basic principle is to scan along the code in the order of parsing/execution, and keep a list of tracked +// variables that are current contenders for elimination. We must untrack when we see something that we cannot +// cross, for example, a write to memory means we must invalidate variables that depend on reading from +// memory, since if we change the order then we do not preserve the computation. +// +// We rely on some assumptions about emscripten-generated code here, which means we can do a lot more than +// a general JS optimization can. For example, we assume that SUB nodes (indexing like HEAP[..]) are +// memory accesses or FUNCTION_TABLE accesses, and in both cases that the symbol cannot be replaced although +// the contents can. So we assume FUNCTION_TABLE might have its contents changed but not be pointed to +// a different object, which allows +// +// var x = f(); +// FUNCTION_TABLE[x](); +// +// to be optimized (f could replace FUNCTION_TABLE, so in general JS eliminating x is not valid). +// +// In memSafe mode, we are more careful and assume functions can replace HEAP and FUNCTION_TABLE, which +// can happen in ALLOW_MEMORY_GROWTH mode + +StringSet ELIMINATION_SAFE_NODES("var assign call if toplevel do return label switch binary unary-prefix"); // do is checked carefully, however +StringSet IGNORABLE_ELIMINATOR_SCAN_NODES("num toplevel string break continue dot"); // dot can only be STRING_TABLE.* +StringSet ABORTING_ELIMINATOR_SCAN_NODES("new object function defun for while array throw"); // we could handle some of these, TODO, but nontrivial (e.g. for while, the condition is hit multiple times after the body) + +bool isTempDoublePtrAccess(Ref node) { // these are used in bitcasts; they are not really affecting memory, and should cause no invalidation + assert(node[0] == SUB); + return (node[2][0] == NAME && node[2][1] == TEMP_DOUBLE_PTR) || + (node[2][0] == BINARY && ((node[2][2][0] == NAME && node[2][2][1] == TEMP_DOUBLE_PTR) || + (node[2][3][0] == NAME && node[2][3][1] == TEMP_DOUBLE_PTR))); +} + +class StringIntMap : public std::unordered_map { +public: + HASES +}; + +class StringRefMap : public std::unordered_map { +public: + HASES +}; + +void eliminate(Ref ast, bool memSafe=false) { + // Find variables that have a single use, and if they can be eliminated, do so + traverseFunctions(ast, [&memSafe](Ref func) { + AsmData asmData(func); + + // First, find the potentially eliminatable functions: that have one definition and one use + + StringIntMap definitions; + StringIntMap uses; + StringIntMap namings; + StringRefMap values; + StringIntMap varsToRemove; // variables being removed, that we can eliminate all 'var x;' of (this refers to VAR nodes we should remove) + // 1 means we should remove it, 2 means we successfully removed it + StringSet varsToTryToRemove; // variables that have 0 uses, but have side effects - when we scan we can try to remove them + + // examine body and note locals + traversePre(func, [&](Ref node) { + IString type = node[0]->getIString(); + assert(type != VAR); + if (type == NAME) { + IString& name = node[1]->getIString(); + uses[name]++;// = uses[name] + 1; + } else if (type == ASSIGN) { + Ref target = node[2]; + if (target[0] == NAME) { + IString& name = target[1]->getIString(); + definitions[name]++;//= definitions[name] + 1; + uses[name]; // zero if not there already + if (!values.has(name)) values[name] = node[3]; + assert(node[1]->isBool(true)); // not +=, -= etc., just = + uses[name]--; // because the name node will show up by itself in the previous case + namings[name]++;// = namings[name] + 1; // offset it here, this tracks the total times we are named + } + } + }); + + for (auto used : uses) { + namings[used.first] += used.second; + } + + StringSet potentials; // local variables with 1 definition and 1 use + StringSet sideEffectFree; // whether a local variable has no side effects in its definition. Only relevant when there are no uses + + auto unprocessVariable = [&](IString name) { + potentials.erase(name); + varsToRemove.erase(name); + sideEffectFree.erase(name); + varsToTryToRemove.erase(name); + }; + std::function processVariable = [&](IString name) { + if (definitions[name] == 1 && uses[name] == 1) { + potentials.insert(name); + } else if (uses[name] == 0 && definitions[name] <= 1) { // no uses, no def or 1 def (cannot operate on phis, and the llvm optimizer will remove unneeded phis anyhow) (no definition means it is a function parameter, or a local with just |var x;| but no defining assignment) + bool sideEffects = false; + Ref value = values[name]; + if (!!value) { + // TODO: merge with other side effect code + // First, pattern-match + // (HEAP32[((tempDoublePtr)>>2)]=((HEAP32[(($_sroa_0_0__idx1)>>2)])|0),HEAP32[(((tempDoublePtr)+(4))>>2)]=((HEAP32[((($_sroa_0_0__idx1)+(4))>>2)])|0),(+(HEAPF64[(tempDoublePtr)>>3]))) + // which has no side effects and is the special form of converting double to i64. + if (!(value[0] == SEQ && value[1][0] == ASSIGN && value[1][2][0] == SUB && value[1][2][2][0] == BINARY && value[1][2][2][1] == RSHIFT && + value[1][2][2][2][0] == NAME && value[1][2][2][2][1] == TEMP_DOUBLE_PTR)) { + // If not that, then traverse and scan normally. + sideEffects = hasSideEffects(value); + } + } + if (!sideEffects) { + varsToRemove[name] = !definitions[name] ? 2 : 1; // remove it normally + sideEffectFree.insert(name); + // Each time we remove a variable with 0 uses, if its value has no + // side effects and vanishes too, then we can remove a use from variables + // appearing in it, and possibly eliminate again + if (!!value) { + traversePre(value, [&](Ref node) { + if (node[0] == NAME) { + IString name = node[1]->getIString(); + node[1]->setString(EMPTY); // we can remove this - it will never be shown, and should not be left to confuse us as we traverse + if (asmData.isLocal(name)) { + uses[name]--; // cannot be infinite recursion since we descend an energy function + assert(uses[name] >= 0); + unprocessVariable(name); + processVariable(name); + } + } + }); + } + } else { + varsToTryToRemove.insert(name); // try to remove it later during scanning + } + } + }; + for (auto name : asmData.locals) { + processVariable(name.first); + } + + //printErr('defs: ' + JSON.stringify(definitions)); + //printErr('uses: ' + JSON.stringify(uses)); + //printErr('values: ' + JSON.stringify(values)); + //printErr('locals: ' + JSON.stringify(locals)); + //printErr('varsToRemove: ' + JSON.stringify(varsToRemove)); + //printErr('varsToTryToRemove: ' + JSON.stringify(varsToTryToRemove)); + values.clear(); + //printErr('potentials: ' + JSON.stringify(potentials)); + // We can now proceed through the function. In each list of statements, we try to eliminate + struct Tracking { + bool usesGlobals, usesMemory; + Ref defNode; + StringSet deps; + bool doesCall; + }; + class Tracked : public std::unordered_map { + public: + HASES + }; + Tracked tracked; + + bool globalsInvalidated = false; // do not repeat invalidations, until we track something new + bool memoryInvalidated = false; + bool callsInvalidated = false; + auto track = [&](IString name, Ref value, Ref defNode) { // add a potential that has just been defined to the tracked list, we hope to eliminate it + Tracking& track = tracked[name]; + track.usesGlobals = false; + track.usesMemory = false; + track.doesCall = false; + bool ignoreName = false; // one-time ignorings of names, as first op in sub and call + traversePre(value, [&](Ref node) { + IString type = node[0]->getIString(); + if (type == NAME) { + if (!ignoreName) { + IString name = node[1]->getIString(); + if (!asmData.isLocal(name)) { + track.usesGlobals = true; + } + if (!potentials.has(name)) { // deps do not matter for potentials - they are defined once, so no complexity + track.deps.insert(name); + } + } else { + ignoreName = false; + } + } else if (type == SUB) { + track.usesMemory = true; + ignoreName = true; + } else if (type == CALL) { + track.usesGlobals = true; + track.usesMemory = true; + track.doesCall = true; + ignoreName = true; + } else { + ignoreName = false; + } + }); + globalsInvalidated = false; + memoryInvalidated = false; + callsInvalidated = false; + }; + + // TODO: invalidate using a sequence number for each type (if you were tracked before the last invalidation, you are cancelled). remove for.in loops + #define INVALIDATE(what, check) \ + auto invalidate##what = [&]() { \ + std::vector temp; \ + for (auto t : tracked) { \ + IString name = t.first; \ + Tracking& info = tracked[name]; \ + if (check) { \ + temp.push_back(name); \ + } \ + } \ + for (int i = 0; i < temp.size(); i++) { \ + tracked.erase(temp[i]); \ + } \ + }; + INVALIDATE(Globals, info.usesGlobals); + INVALIDATE(Memory, info.usesMemory); + INVALIDATE(Calls, info.doesCall); + + auto invalidateByDep = [&](IString dep) { + std::vector temp; + for (auto t : tracked) { + IString name = t.first; + Tracking& info = tracked[name]; + if (info.deps.has(dep)) { + temp.push_back(name); + } + } + for (int i = 0; i < temp.size(); i++) { + tracked.erase(temp[i]); + } + }; + + std::function doEliminate; + + // Generate the sequence of execution. This determines what is executed before what, so we know what can be reordered. Using + // that, performs invalidations and eliminations + auto scan = [&](Ref node) { + bool abort = false; + bool allowTracking = true; // false inside an if; also prevents recursing in an if + std::function traverseInOrder = [&](Ref node, bool ignoreSub, bool ignoreName) { + if (abort) return; + IString type = node[0]->getIString(); + if (type == ASSIGN) { + Ref target = node[2]; + Ref value = node[3]; + bool nameTarget = target[0] == NAME; + traverseInOrder(target, true, nameTarget); // evaluate left + traverseInOrder(value, false, false); // evaluate right + // do the actual assignment + if (nameTarget) { + IString name = target[1]->getIString(); + if (!potentials.has(name)) { + if (!varsToTryToRemove.has(name)) { + // expensive check for invalidating specific tracked vars. This list is generally quite short though, because of + // how we just eliminate in short spans and abort when control flow happens TODO: history numbers instead + invalidateByDep(name); // can happen more than once per dep.. + if (!asmData.isLocal(name) && !globalsInvalidated) { + invalidateGlobals(); + globalsInvalidated = true; + } + // if we can track this name (that we assign into), and it has 0 uses and we want to remove its VAR + // definition - then remove it right now, there is no later chance + if (allowTracking && varsToRemove.has(name) && uses[name] == 0) { + track(name, node[3], node); + doEliminate(name, node); + } + } else { + // replace it in-place + safeCopy(node, value); + varsToRemove[name] = 2; + } + } else { + if (allowTracking) track(name, node[3], node); + } + } else if (target[0] == SUB) { + if (isTempDoublePtrAccess(target)) { + if (!globalsInvalidated) { + invalidateGlobals(); + globalsInvalidated = true; + } + } else if (!memoryInvalidated) { + invalidateMemory(); + memoryInvalidated = true; + } + } + } else if (type == SUB) { + traverseInOrder(node[1], false, !memSafe); // evaluate inner + traverseInOrder(node[2], false, false); // evaluate outer + // ignoreSub means we are a write (happening later), not a read + if (!ignoreSub && !isTempDoublePtrAccess(node)) { + // do the memory access + if (!callsInvalidated) { + invalidateCalls(); + callsInvalidated = true; + } + } + } else if (type == VAR) { + Ref vars = node[1]; + for (int i = 0; i < vars->size(); i++) { + IString name = vars[i][0]->getIString(); + Ref value = vars[i][1]; + if (!!value) { + traverseInOrder(value, false, false); + if (potentials.has(name) && allowTracking) { + track(name, value, node); + } else { + invalidateByDep(name); + } + if (vars->size() == 1 && varsToTryToRemove.has(name) && !!value) { + // replace it in-place + value = make1(STAT, value); + safeCopy(node, value); + varsToRemove[name] = 2; + } + } + } + } else if (type == BINARY) { + bool flipped = false; + if (ASSOCIATIVE_BINARIES.has(node[1]) && !NAME_OR_NUM.has(node[2][0]) && NAME_OR_NUM.has(node[3][0])) { // TODO recurse here? + // associatives like + and * can be reordered in the simple case of one of the sides being a name, since we assume they are all just numbers + Ref temp = node[2]; + node[2] = node[3]; + node[3] = temp; + flipped = true; + } + traverseInOrder(node[2], false, false); + traverseInOrder(node[3], false, false); + if (flipped && NAME_OR_NUM.has(node[2][0])) { // dunno if we optimized, but safe to flip back - and keeps the code closer to the original and more readable + Ref temp = node[2]; + node[2] = node[3]; + node[3] = temp; + } + } else if (type == NAME) { + if (!ignoreName) { // ignoreName means we are the name of something like a call or a sub - irrelevant for us + IString name = node[1]->getIString(); + if (tracked.has(name)) { + doEliminate(name, node); + } else if (!asmData.isLocal(name) && !callsInvalidated) { + invalidateCalls(); + callsInvalidated = true; + } + } + } else if (type == UNARY_PREFIX) { //|| type == 'unary-postfix') { + traverseInOrder(node[2], false, false); + } else if (IGNORABLE_ELIMINATOR_SCAN_NODES.has(type)) { + } else if (type == CALL) { + traverseInOrder(node[1], false, true); + Ref args = node[2]; + for (int i = 0; i < args->size(); i++) { + traverseInOrder(args[i], false, false); + } + if (callHasSideEffects(node)) { + // these two invalidations will also invalidate calls + if (!globalsInvalidated) { + invalidateGlobals(); + globalsInvalidated = true; + } + if (!memoryInvalidated) { + invalidateMemory(); + memoryInvalidated = true; + } + } + } else if (type == IF) { + if (allowTracking) { + traverseInOrder(node[1], false, false); // can eliminate into condition, but nowhere else + if (!callsInvalidated) { // invalidate calls, since we cannot eliminate them into an if that may not execute! + invalidateCalls(); + callsInvalidated = true; + } + allowTracking = false; + traverseInOrder(node[2], false, false); // 2 and 3 could be 'parallel', really.. + if (!!node[3]) traverseInOrder(node[3], false, false); + allowTracking = true; + } else { + tracked.clear(); + } + } else if (type == BLOCK) { + Ref stats = node[1]; + if (!!stats) { + for (int i = 0; i < stats->size(); i++) { + traverseInOrder(stats[i], false, false); + } + } + } else if (type == STAT) { + traverseInOrder(node[1], false, false); + } else if (type == LABEL) { + traverseInOrder(node[2], false, false); + } else if (type == SEQ) { + traverseInOrder(node[1], false, false); + traverseInOrder(node[2], false, false); + } else if (type == DO) { + if (node[1][0] == NUM && node[1][1]->getNumber() == 0) { // one-time loop + traverseInOrder(node[2], false, false); + } else { + tracked.clear(); + } + } else if (type == RETURN) { + if (!!node[1]) traverseInOrder(node[1], false, false); + } else if (type == CONDITIONAL) { + if (!callsInvalidated) { // invalidate calls, since we cannot eliminate them into a branch of an LLVM select/JS conditional that does not execute + invalidateCalls(); + callsInvalidated = true; + } + traverseInOrder(node[1], false, false); + traverseInOrder(node[2], false, false); + traverseInOrder(node[3], false, false); + } else if (type == SWITCH) { + traverseInOrder(node[1], false, false); + StringSet originalTracked; + for (auto t : tracked) originalTracked.insert(t.first); + Ref cases = node[2]; + for (int i = 0; i < cases->size(); i++) { + Ref c = cases[i]; + assert(c[0]->isNull() || c[0][0] == NUM || (c[0][0] == UNARY_PREFIX && c[0][2][0] == NUM)); + Ref stats = c[1]; + for (int j = 0; j < stats->size(); j++) { + traverseInOrder(stats[j], false, false); + } + // We cannot track from one switch case into another, undo all new trackings TODO: general framework here, use in if-else as well + for (auto t : tracked) { + IString name = t.first; + if (!originalTracked.has(name)) { + Tracking& info = tracked[name]; + if (info.usesGlobals || info.usesMemory || info.deps.size() > 0) { + tracked.erase(name); + } + } + } + } + tracked.clear(); // do not track from inside the switch to outside + } else { + assert(ABORTING_ELIMINATOR_SCAN_NODES.has(type)); + tracked.clear(); + abort = true; + } + }; + traverseInOrder(node, false, false); + }; + //var eliminationLimit = 0; // used to debugging purposes + doEliminate = [&](IString name, Ref node) { + //if (eliminationLimit == 0) return; + //eliminationLimit--; + //printErr('elim!!!!! ' + name); + // yes, eliminate! + varsToRemove[name] = 2; // both assign and var definitions can have other vars we must clean up + Tracking& info = tracked[name]; + Ref defNode = info.defNode; + if (!sideEffectFree.has(name)) { + assert(defNode[0] != VAR); + // assign + Ref value = defNode[3]; + // wipe out the assign + safeCopy(defNode, makeEmpty()); + // replace this node in-place + safeCopy(node, value); + } else { + // This has no side effects and no uses, empty it out in-place + safeCopy(node, makeEmpty()); + } + tracked.erase(name); + }; + traversePre(func, [&](Ref block) { + // Look for statements, including while-switch pattern + Ref stats = getStatements(block); + if (!stats && (block[0] == WHILE && block[2][0] == SWITCH)) { + stats = &(makeArray()->push_back(block[2])); + } + if (!stats) return; + tracked.clear(); + for (int i = 0; i < stats->size(); i++) { + Ref node = deStat(stats[i]); + IString type = node[0]->getIString(); + if (type == RETURN && i < stats->size()-1) { + stats->setSize(i+1); // remove any code after a return + } + // Check for things that affect elimination + if (ELIMINATION_SAFE_NODES.has(type)) { + scan(node); + } else { + tracked.clear(); // not a var or assign, break all potential elimination so far + } + } + }); + + //var seenUses = {}, helperReplacements = {}; // for looper-helper optimization + + // clean up vars, and loop variable elimination + traversePre(func, [&](Ref node) { + // pre + IString type = node[0]->getIString(); + assert(type != VAR); + if (type == ASSIGN && node[1]->isBool(true) && node[2][0] == NAME && node[3][0] == NAME && node[2][1] == node[3][1]) { + // elimination led to X = X, which we can just remove + safeCopy(node, makeEmpty()); + } + }); /*, function(node, type) { + // post + if (type == NAME) { + var name = node[1]; + if (name in helperReplacements) { + node[1] = helperReplacements[name]; + return; // no need to track this anymore, we can't loop-optimize more than once + } + // track how many uses we saw. we need to know when a variable is no longer used (hence we run this in the post) + if (!(name in seenUses)) { + seenUses[name] = 1; + } else { + seenUses[name]++; + } + } else if (type == WHILE) { + if (!asm) return; + // try to remove loop helper variables specifically + var stats = node[2][1]; + var last = stats[stats.length-1]; + if (last && last[0] == IF && last[2][0] == BLOCK && last[3] && last[3][0] == BLOCK) { + var ifTrue = last[2]; + var ifFalse = last[3]; + clearEmptyNodes(ifTrue[1]); + clearEmptyNodes(ifFalse[1]); + var flip = false; + if (ifFalse[1][0] && ifFalse[1][ifFalse[1].length-1][0] == 'break') { // canonicalize break in the if-true + var temp = ifFalse; + ifFalse = ifTrue; + ifTrue = temp; + flip = true; + } + if (ifTrue[1][0] && ifTrue[1][ifTrue[1].length-1][0] == 'break') { + var assigns = ifFalse[1]; + clearEmptyNodes(assigns); + var loopers = [], helpers = []; + for (var i = 0; i < assigns.length; i++) { + if (assigns[i][0] == STAT && assigns[i][1][0] == ASSIGN) { + var assign = assigns[i][1]; + if (assign[1] == true && assign[2][0] == NAME && assign[3][0] == NAME) { + var looper = assign[2][1]; + var helper = assign[3][1]; + if (definitions[helper] == 1 && seenUses[looper] == namings[looper] && + !helperReplacements[helper] && !helperReplacements[looper]) { + loopers.push(looper); + helpers.push(helper); + } + } + } + } + // remove loop vars that are used in the rest of the else + for (var i = 0; i < assigns.length; i++) { + if (assigns[i][0] == STAT && assigns[i][1][0] == ASSIGN) { + var assign = assigns[i][1]; + if (!(assign[1] == true && assign[2][0] == NAME && assign[3][0] == NAME) || loopers.indexOf(assign[2][1]) < 0) { + // this is not one of the loop assigns + traverse(assign, function(node, type) { + if (type == NAME) { + var index = loopers.indexOf(node[1]); + if (index < 0) index = helpers.indexOf(node[1]); + if (index >= 0) { + loopers.splice(index, 1); + helpers.splice(index, 1); + } + } + }); + } + } + } + // remove loop vars that are used in the if + traverse(ifTrue, function(node, type) { + if (type == NAME) { + var index = loopers.indexOf(node[1]); + if (index < 0) index = helpers.indexOf(node[1]); + if (index >= 0) { + loopers.splice(index, 1); + helpers.splice(index, 1); + } + } + }); + if (loopers.length == 0) return; + for (var l = 0; l < loopers.length; l++) { + var looper = loopers[l]; + var helper = helpers[l]; + // the remaining issue is whether loopers are used after the assignment to helper and before the last line (where we assign to it) + var found = -1; + for (var i = stats.length-2; i >= 0; i--) { + var curr = stats[i]; + if (curr[0] == STAT && curr[1][0] == ASSIGN) { + var currAssign = curr[1]; + if (currAssign[1] == true && currAssign[2][0] == NAME) { + var to = currAssign[2][1]; + if (to == helper) { + found = i; + break; + } + } + } + } + if (found < 0) return; + // if a loop variable is used after we assigned to the helper, we must save its value and use that. + // (note that this can happen due to elimination, if we eliminate an expression containing the + // loop var far down, past the assignment!) + // first, see if the looper and helpers overlap. Note that we check for this looper, compared to + // *ALL* the helpers. Helpers will be replaced by loopers as we eliminate them, potentially + // causing conflicts, so any helper is a concern. + var firstLooperUsage = -1; + var lastLooperUsage = -1; + var firstHelperUsage = -1; + for (var i = found+1; i < stats.length; i++) { + var curr = i < stats.length-1 ? stats[i] : last[1]; // on the last line, just look in the condition + traverse(curr, function(node, type) { + if (type == NAME) { + if (node[1] == looper) { + if (firstLooperUsage < 0) firstLooperUsage = i; + lastLooperUsage = i; + } else if (helpers.indexOf(node[1]) >= 0) { + if (firstHelperUsage < 0) firstHelperUsage = i; + } + } + }); + } + if (firstLooperUsage >= 0) { + // the looper is used, we cannot simply merge the two variables + if ((firstHelperUsage < 0 || firstHelperUsage > lastLooperUsage) && lastLooperUsage+1 < stats.length && triviallySafeToMove(stats[found], asmData) && + seenUses[helper] == namings[helper]) { + // the helper is not used, or it is used after the last use of the looper, so they do not overlap, + // and the last looper usage is not on the last line (where we could not append after it), and the + // helper is not used outside of the loop. + // just move the looper definition to after the looper's last use + stats.splice(lastLooperUsage+1, 0, stats[found]); + stats.splice(found, 1); + } else { + // they overlap, we can still proceed with the loop optimization, but we must introduce a + // loop temp helper variable + var temp = looper + '$looptemp'; + assert(!(temp in asmData.vars)); + for (var i = firstLooperUsage; i <= lastLooperUsage; i++) { + var curr = i < stats.length-1 ? stats[i] : last[1]; // on the last line, just look in the condition + traverse(curr, function looperToLooptemp(node, type) { + if (type == NAME) { + if (node[1] == looper) { + node[1] = temp; + } + } else if (type == ASSIGN && node[2][0] == NAME) { + // do not traverse the assignment target, phi assignments to the loop variable must remain + traverse(node[3], looperToLooptemp); + return null; + } + }); + } + asmData.vars[temp] = asmData.vars[looper]; + stats.splice(found, 0, [STAT, [ASSIGN, true, [NAME, temp], [NAME, looper]]]); + } + } + } + for (var l = 0; l < helpers.length; l++) { + for (var k = 0; k < helpers.length; k++) { + if (l != k && helpers[l] == helpers[k]) return; // it is complicated to handle a shared helper, abort + } + } + // hurrah! this is safe to do + //printErr("ELIM LOOP VAR " + JSON.stringify(loopers) + ' :: ' + JSON.stringify(helpers)); + for (var l = 0; l < loopers.length; l++) { + var looper = loopers[l]; + var helper = helpers[l]; + varsToRemove[helper] = 2; + traverse(node, function(node, type) { // replace all appearances of helper with looper + if (type == NAME && node[1] == helper) node[1] = looper; + }); + helperReplacements[helper] = looper; // replace all future appearances of helper with looper + helperReplacements[looper] = looper; // avoid any further attempts to optimize looper in this manner (seenUses is wrong anyhow, too) + } + // simplify the if. we remove the if branch, leaving only the else + if (flip) { + last[1] = simplifyNotCompsDirect([UNARY_PREFIX, '!', last[1]]); + var temp = last[2]; + last[2] = last[3]; + last[3] = temp; + } + if (loopers.length == assigns.length) { + last.pop(); + } else { + var elseStats = getStatements(last[3]); + for (var i = 0; i < elseStats.length; i++) { + var stat = elseStats[i]; + if (stat[0] == STAT) stat = stat[1]; + if (stat[0] == ASSIGN && stat[2][0] == NAME) { + if (loopers.indexOf(stat[2][1]) >= 0) { + elseStats[i] = emptyNode(); + } + } + } + } + } + } + } + }); */ + + for (auto v : varsToRemove) { + if (v.second == 2) asmData.deleteVar(v.first); + } + + asmData.denormalize(); + }); + + removeAllEmptySubNodes(ast); +} + +void eliminateMemSafe(Ref ast) { + eliminate(ast, true); +} + void simplifyExpressions(Ref ast) { // Simplify common expressions used to perform integer conversion operations // in cases where no conversion is needed. @@ -1435,6 +2193,8 @@ int main(int argc, char **argv) { if (str == "asm") {} // the default for us else if (str == "asmPreciseF32") preciseF32 = true; else if (str == "receiveJSON" || str == "emitJSON") {} // the default for us + else if (str == "eliminate") eliminate(doc); + else if (str == "eliminateMemSafe") eliminateMemSafe(doc); else if (str == "simplifyExpressions") simplifyExpressions(doc); else if (str == "optimizeFrounds") optimizeFrounds(doc); else if (str == "simplifyIfs") simplifyIfs(doc); From d1905691b9e9a756e3d047388af5cd69d1163f0d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Nov 2014 15:24:18 -0800 Subject: [PATCH 098/161] ignore non-asm tests for new optimizer --- tests/test_other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_other.py b/tests/test_other.py index eb5d79871a342..c99dd14c3ab12 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -1938,7 +1938,7 @@ def test_js_optimizer(self): print ' js' output = Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), input] + passes, stdin=PIPE, stdout=PIPE).communicate()[0] self.assertIdentical(expected, output.replace('\r\n', '\n').replace('\n\n', '\n')) - if js_optimizer.use_native(passes): + if js_optimizer.use_native(passes) and 'asm' in passes: # test calling native print ' native' self.clear() From a66b203746e43fe33a566f2eaf2c22fa91686cc1 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Nov 2014 15:24:33 -0800 Subject: [PATCH 099/161] fix bug in asm-eliminator test --- tools/eliminator/asm-eliminator-test-output.js | 2 +- tools/eliminator/asm-eliminator-test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/eliminator/asm-eliminator-test-output.js b/tools/eliminator/asm-eliminator-test-output.js index eafb3b797e4a2..e3dd0113d2a90 100644 --- a/tools/eliminator/asm-eliminator-test-output.js +++ b/tools/eliminator/asm-eliminator-test-output.js @@ -298,7 +298,7 @@ function tempDouble2($46, $14, $28, $42, $20, $32, $45) { $20 = $20 | 0; $32 = $32 | 0; $45 = $45 | 0; - var $46 = 0, $_sroa_06_0_insert_insert$1 = 0; + var $_sroa_06_0_insert_insert$1 = 0; $46 = (HEAPF32[tempDoublePtr >> 2] = ($14 < $28 ? $14 : $28) - $42, HEAP32[tempDoublePtr >> 2] | 0); $_sroa_06_0_insert_insert$1 = (HEAPF32[tempDoublePtr >> 2] = ($20 < $32 ? $20 : $32) - $42, HEAP32[tempDoublePtr >> 2] | 0) | 0; HEAP32[$45 >> 2] = 0 | $46; diff --git a/tools/eliminator/asm-eliminator-test.js b/tools/eliminator/asm-eliminator-test.js index 1ce8aaed9e52d..3624536a42471 100644 --- a/tools/eliminator/asm-eliminator-test.js +++ b/tools/eliminator/asm-eliminator-test.js @@ -374,7 +374,7 @@ function tempDouble2($46, $14, $28, $42, $20, $32, $45) { $20 = $20 | 0; $32 = $32 | 0; $45 = $45 | 0; - var $46 = 0, $_sroa_06_0_insert_insert$1 = 0; + var $_sroa_06_0_insert_insert$1 = 0; $46 = (HEAPF32[tempDoublePtr >> 2] = ($14 < $28 ? $14 : $28) - $42, HEAP32[tempDoublePtr >> 2] | 0); $_sroa_06_0_insert_insert$1 = (HEAPF32[tempDoublePtr >> 2] = ($20 < $32 ? $20 : $32) - $42, HEAP32[tempDoublePtr >> 2] | 0) | 0; HEAP32[$45 >> 2] = 0 | $46; From 3d6d6ed1ef3279bd798b9df1fdecbc3f370e4b76 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Nov 2014 15:24:59 -0800 Subject: [PATCH 100/161] fix Ref not operator --- tools/optimizer/minijson.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index e7bd7a1ef5363..f4128fb320c2f 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -497,7 +497,7 @@ bool Ref::operator==(Ref other) { } bool Ref::operator!() { - return get()->isNull(); + return !get() || get()->isNull(); } // Arena methods From 88d8f9687d58f72a34a592a03335defdb18b1c9b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Nov 2014 15:25:23 -0800 Subject: [PATCH 101/161] eliminator fixes --- tools/optimizer/optimizer.cpp | 61 ++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 8e1d373000d57..b1f0a8ddc18c1 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -36,6 +36,7 @@ IString TOPLEVEL("toplevel"), NaN("nan"), TEMP_RET0("tempRet0"), UNARY_PREFIX("unary-prefix"), + UNARY_POSTFIX("unary-postfix"), MATH_FROUND("Math_fround"), SIMD_FLOAT32X4("SIMD_float32x4"), SIMD_INT32X4("SIMD_int32x4"), @@ -891,9 +892,20 @@ void eliminate(Ref ast, bool memSafe=false) { // examine body and note locals traversePre(func, [&](Ref node) { - IString type = node[0]->getIString(); - assert(type != VAR); - if (type == NAME) { + Ref type = node[0]; + if (type == VAR) { + Ref node1 = node[1]; + for (int i = 0; i < node1->size(); i++) { + Ref node1i = node1[i]; + IString name = node1i[0]->getIString(); + Ref value = node1i[1]; + if (!!value) { + definitions[name]++; + if (!values.has(name)) values[name] = value; + } + uses[name]; + } + } else if (type == NAME) { IString& name = node[1]->getIString(); uses[name]++;// = uses[name] + 1; } else if (type == ASSIGN) { @@ -928,8 +940,10 @@ void eliminate(Ref ast, bool memSafe=false) { potentials.insert(name); } else if (uses[name] == 0 && definitions[name] <= 1) { // no uses, no def or 1 def (cannot operate on phis, and the llvm optimizer will remove unneeded phis anyhow) (no definition means it is a function parameter, or a local with just |var x;| but no defining assignment) bool sideEffects = false; - Ref value = values[name]; - if (!!value) { + auto val = values.find(name); + Ref value; + if (val != values.end()) { + value = val->second; // TODO: merge with other side effect code // First, pattern-match // (HEAP32[((tempDoublePtr)>>2)]=((HEAP32[(($_sroa_0_0__idx1)>>2)])|0),HEAP32[(((tempDoublePtr)+(4))>>2)]=((HEAP32[((($_sroa_0_0__idx1)+(4))>>2)])|0),(+(HEAPF64[(tempDoublePtr)>>3]))) @@ -997,10 +1011,11 @@ void eliminate(Ref ast, bool memSafe=false) { Tracking& track = tracked[name]; track.usesGlobals = false; track.usesMemory = false; + track.defNode = defNode; track.doesCall = false; bool ignoreName = false; // one-time ignorings of names, as first op in sub and call traversePre(value, [&](Ref node) { - IString type = node[0]->getIString(); + Ref type = node[0]; if (type == NAME) { if (!ignoreName) { IString name = node[1]->getIString(); @@ -1072,7 +1087,7 @@ void eliminate(Ref ast, bool memSafe=false) { bool allowTracking = true; // false inside an if; also prevents recursing in an if std::function traverseInOrder = [&](Ref node, bool ignoreSub, bool ignoreName) { if (abort) return; - IString type = node[0]->getIString(); + Ref type = node[0]; if (type == ASSIGN) { Ref target = node[2]; Ref value = node[3]; @@ -1173,7 +1188,7 @@ void eliminate(Ref ast, bool memSafe=false) { callsInvalidated = true; } } - } else if (type == UNARY_PREFIX) { //|| type == 'unary-postfix') { + } else if (type == UNARY_PREFIX || type == UNARY_POSTFIX) { traverseInOrder(node[2], false, false); } else if (IGNORABLE_ELIMINATOR_SCAN_NODES.has(type)) { } else if (type == CALL) { @@ -1239,8 +1254,7 @@ void eliminate(Ref ast, bool memSafe=false) { traverseInOrder(node[3], false, false); } else if (type == SWITCH) { traverseInOrder(node[1], false, false); - StringSet originalTracked; - for (auto t : tracked) originalTracked.insert(t.first); + Tracked originalTracked = tracked; Ref cases = node[2]; for (int i = 0; i < cases->size(); i++) { Ref c = cases[i]; @@ -1250,15 +1264,7 @@ void eliminate(Ref ast, bool memSafe=false) { traverseInOrder(stats[j], false, false); } // We cannot track from one switch case into another, undo all new trackings TODO: general framework here, use in if-else as well - for (auto t : tracked) { - IString name = t.first; - if (!originalTracked.has(name)) { - Tracking& info = tracked[name]; - if (info.usesGlobals || info.usesMemory || info.deps.size() > 0) { - tracked.erase(name); - } - } - } + tracked = originalTracked; // TODO: don't do this on the last one } tracked.clear(); // do not track from inside the switch to outside } else { @@ -1276,8 +1282,10 @@ void eliminate(Ref ast, bool memSafe=false) { //printErr('elim!!!!! ' + name); // yes, eliminate! varsToRemove[name] = 2; // both assign and var definitions can have other vars we must clean up + assert(tracked.has(name)); Tracking& info = tracked[name]; Ref defNode = info.defNode; + assert(!!defNode); if (!sideEffectFree.has(name)) { assert(defNode[0] != VAR); // assign @@ -1302,7 +1310,7 @@ void eliminate(Ref ast, bool memSafe=false) { tracked.clear(); for (int i = 0; i < stats->size(); i++) { Ref node = deStat(stats[i]); - IString type = node[0]->getIString(); + Ref type = node[0]; if (type == RETURN && i < stats->size()-1) { stats->setSize(i+1); // remove any code after a return } @@ -1320,8 +1328,15 @@ void eliminate(Ref ast, bool memSafe=false) { // clean up vars, and loop variable elimination traversePre(func, [&](Ref node) { // pre - IString type = node[0]->getIString(); - assert(type != VAR); + Ref type = node[0]; + /*if (type === VAR) { + node[1] = node[1].filter(function(pair) { return !varsToRemove[pair[0]] }); + if (node[1].length === 0) { + // wipe out an empty |var;| + node[0] = 'toplevel'; + node[1] = []; + } + } else */ if (type == ASSIGN && node[1]->isBool(true) && node[2][0] == NAME && node[3][0] == NAME && node[2][1] == node[3][1]) { // elimination led to X = X, which we can just remove safeCopy(node, makeEmpty()); @@ -1906,7 +1921,7 @@ void simplifyExpressions(Ref ast) { }; std::function emitsBoolean = [&emitsBoolean](Ref node) { - IString type = node[0]->getIString(); + Ref type = node[0]; if (type == NUM) { return node[1]->getNumber() == 0 || node[1]->getNumber() == 1; } From d97b1c63844b3915d3863700adaae773b86690de Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Nov 2014 16:36:27 -0800 Subject: [PATCH 102/161] get all of ported eliminator to build --- tools/optimizer/minijson.h | 9 ++ tools/optimizer/optimizer.cpp | 260 ++++++++++++++++++++-------------- 2 files changed, 159 insertions(+), 110 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index f4128fb320c2f..0c614842b4157 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -439,6 +439,12 @@ struct Value { arr->push_back(r); return *this; } + Ref pop_back() { + assert(isArray()); + Ref ret = arr->back(); + arr->pop_back(); + return ret; + } void splice(int x, int num) { assert(isArray()); @@ -448,6 +454,9 @@ struct Value { void insert(int x, int num) { arr->insert(arr->begin() + x, num, Ref()); } + void insert(int x, Ref node) { + arr->insert(arr->begin() + x, 1, node); + } int indexOf(Ref other) { assert(isArray()); diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index b1f0a8ddc18c1..c9ec4ed176796 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -30,6 +30,8 @@ IString TOPLEVEL("toplevel"), CALL("call"), NUM("num"), LABEL("label"), + BREAK("break"), + CONTINUE("continue"), SWITCH("switch"), STRING("string"), INF("inf"), @@ -74,6 +76,14 @@ IString TOPLEVEL("toplevel"), // Infrastructure //================== +template +int indexOf(T list, V value) { + for (int i = 0; i < list.size(); i++) { + if (list[i] == value) return i; + } + return -1; +} + int jsD2I(double x) { return (int)((int64_t)x); } @@ -371,6 +381,11 @@ struct AsmData { //printErr('denormalized \n\n' + astToSrc(func) + '\n\n'); } + void addVar(IString name, AsmType type) { + locals[name] = { type, false }; + vars.push_back(name); + } + void deleteVar(IString name) { locals.erase(name); for (int i = 0; i < vars.size(); i++) { @@ -670,6 +685,24 @@ bool hasSideEffects(Ref node) { // this is 99% incomplete! return true; } +// checks if a node has just basic operations, nothing with side effects nor that can notice side effects, which +// implies we can move it around in the code +bool triviallySafeToMove(Ref node, AsmData& asmData) { + bool ok = true; + traversePre(node, [&](Ref node) { + Ref type = node[0]; + if (type == STAT || type == BINARY || type == UNARY_PREFIX || type == ASSIGN || type == NUM) return; + else if (type == NAME) { + if (!asmData.isLocal(node[1]->getIString())) ok = false; + } else if (type == CALL) { + if (callHasSideEffects(node)) ok = false; + } else { + ok = false; + } + }); + return ok; +} + // Transforms // We often have branchings that are simplified so one end vanishes, and @@ -719,26 +752,26 @@ void safeCopy(Ref target, Ref source) { // safely copy source onto target, even *target = *temp; } +void clearEmptyNodes(Ref arr) { + int skip = 0; + for (int i = 0; i < arr->size(); i++) { + if (skip) { + arr[i-skip] = arr[i]; + } + if (isEmpty(deStat(arr[i]))) { + skip++; + } + } + if (skip) arr->setSize(arr->size() - skip); +} + void removeAllEmptySubNodes(Ref ast) { traversePre(ast, [](Ref node) { int index = -1; if (node[0] == DEFUN) { - index = 3; + clearEmptyNodes(node[3]); } else if (node[0] == BLOCK && !!node[1]) { - index = 1; - } - if (index > 0) { - Ref arr = node[index]; - int skip = 0; - for (int i = 0; i < arr->size(); i++) { - if (skip) { - arr[i-skip] = arr[i]; - } - if (isEmpty(deStat(arr[i]))) { - skip++; - } - } - if (skip) arr->setSize(arr->size() - skip); + clearEmptyNodes(node[1]); } else if (node[0] == SEQ && isEmpty(node[1])) { safeCopy(node, node[2]); } @@ -870,6 +903,11 @@ class StringIntMap : public std::unordered_map { HASES }; +class StringStringMap : public std::unordered_map { +public: + HASES +}; + class StringRefMap : public std::unordered_map { public: HASES @@ -1323,10 +1361,11 @@ void eliminate(Ref ast, bool memSafe=false) { } }); - //var seenUses = {}, helperReplacements = {}; // for looper-helper optimization + StringIntMap seenUses; + StringStringMap helperReplacements; // for looper-helper optimization // clean up vars, and loop variable elimination - traversePre(func, [&](Ref node) { + traversePrePost(func, [&](Ref node) { // pre Ref type = node[0]; /*if (type === VAR) { @@ -1341,68 +1380,68 @@ void eliminate(Ref ast, bool memSafe=false) { // elimination led to X = X, which we can just remove safeCopy(node, makeEmpty()); } - }); /*, function(node, type) { + }, [&](Ref node) { // post + Ref type = node[0]; if (type == NAME) { - var name = node[1]; - if (name in helperReplacements) { - node[1] = helperReplacements[name]; + IString name = node[1]->getIString(); + if (helperReplacements.has(name)) { + node[1]->setString(helperReplacements[name]); return; // no need to track this anymore, we can't loop-optimize more than once } // track how many uses we saw. we need to know when a variable is no longer used (hence we run this in the post) - if (!(name in seenUses)) { + if (seenUses.has(name)) { seenUses[name] = 1; } else { seenUses[name]++; } } else if (type == WHILE) { - if (!asm) return; // try to remove loop helper variables specifically - var stats = node[2][1]; - var last = stats[stats.length-1]; - if (last && last[0] == IF && last[2][0] == BLOCK && last[3] && last[3][0] == BLOCK) { - var ifTrue = last[2]; - var ifFalse = last[3]; + Ref stats = node[2][1]; + Ref last = stats[stats->size()-1]; + if (!!last && last[0] == IF && last[2][0] == BLOCK && !!last[3] && last[3][0] == BLOCK) { + Ref ifTrue = last[2]; + Ref ifFalse = last[3]; clearEmptyNodes(ifTrue[1]); clearEmptyNodes(ifFalse[1]); - var flip = false; - if (ifFalse[1][0] && ifFalse[1][ifFalse[1].length-1][0] == 'break') { // canonicalize break in the if-true - var temp = ifFalse; + bool flip = false; + if (!!ifFalse[1][0] && ifFalse[1][ifFalse[1]->size()-1][0] == BREAK) { // canonicalize break in the if-true + Ref temp = ifFalse; ifFalse = ifTrue; ifTrue = temp; flip = true; } - if (ifTrue[1][0] && ifTrue[1][ifTrue[1].length-1][0] == 'break') { - var assigns = ifFalse[1]; + if (!!ifTrue[1][0] && ifTrue[1][ifTrue[1]->size()-1][0] == BREAK) { + Ref assigns = ifFalse[1]; clearEmptyNodes(assigns); - var loopers = [], helpers = []; - for (var i = 0; i < assigns.length; i++) { + std::vector loopers, helpers; + for (int i = 0; i < assigns->size(); i++) { if (assigns[i][0] == STAT && assigns[i][1][0] == ASSIGN) { - var assign = assigns[i][1]; - if (assign[1] == true && assign[2][0] == NAME && assign[3][0] == NAME) { - var looper = assign[2][1]; - var helper = assign[3][1]; + Ref assign = assigns[i][1]; + if (assign[1]->isBool(true) && assign[2][0] == NAME && assign[3][0] == NAME) { + IString looper = assign[2][1]->getIString(); + IString helper = assign[3][1]->getIString(); if (definitions[helper] == 1 && seenUses[looper] == namings[looper] && - !helperReplacements[helper] && !helperReplacements[looper]) { - loopers.push(looper); - helpers.push(helper); + !helperReplacements.has(helper) && !helperReplacements.has(looper)) { + loopers.push_back(looper); + helpers.push_back(helper); } } } } // remove loop vars that are used in the rest of the else - for (var i = 0; i < assigns.length; i++) { + for (int i = 0; i < assigns->size(); i++) { if (assigns[i][0] == STAT && assigns[i][1][0] == ASSIGN) { - var assign = assigns[i][1]; - if (!(assign[1] == true && assign[2][0] == NAME && assign[3][0] == NAME) || loopers.indexOf(assign[2][1]) < 0) { + Ref assign = assigns[i][1]; + if (!(assign[1]->isBool(true) && assign[2][0] == NAME && assign[3][0] == NAME) || indexOf(loopers, assign[2][1]->getIString()) < 0) { // this is not one of the loop assigns - traverse(assign, function(node, type) { - if (type == NAME) { - var index = loopers.indexOf(node[1]); - if (index < 0) index = helpers.indexOf(node[1]); + traversePre(assign, [&](Ref node) { + if (node[0] == NAME) { + int index = indexOf(loopers, node[1]->getIString()); + if (index < 0) index = indexOf(helpers, node[1]->getIString()); if (index >= 0) { - loopers.splice(index, 1); - helpers.splice(index, 1); + loopers.erase(loopers.begin() + index); + helpers.erase(helpers.begin() + index); } } }); @@ -1410,28 +1449,28 @@ void eliminate(Ref ast, bool memSafe=false) { } } // remove loop vars that are used in the if - traverse(ifTrue, function(node, type) { - if (type == NAME) { - var index = loopers.indexOf(node[1]); - if (index < 0) index = helpers.indexOf(node[1]); + traversePre(ifTrue, [&](Ref node) { + if (node[0] == NAME) { + int index = indexOf(loopers, node[1]->getIString()); + if (index < 0) index = indexOf(helpers, node[1]->getIString()); if (index >= 0) { - loopers.splice(index, 1); - helpers.splice(index, 1); + loopers.erase(loopers.begin() + index); + helpers.erase(helpers.begin() + index); } } }); - if (loopers.length == 0) return; - for (var l = 0; l < loopers.length; l++) { - var looper = loopers[l]; - var helper = helpers[l]; + if (loopers.size() == 0) return; + for (int l = 0; l < loopers.size(); l++) { + IString looper = loopers[l]; + IString helper = helpers[l]; // the remaining issue is whether loopers are used after the assignment to helper and before the last line (where we assign to it) - var found = -1; - for (var i = stats.length-2; i >= 0; i--) { - var curr = stats[i]; + int found = -1; + for (int i = stats->size()-2; i >= 0; i--) { + Ref curr = stats[i]; if (curr[0] == STAT && curr[1][0] == ASSIGN) { - var currAssign = curr[1]; - if (currAssign[1] == true && currAssign[2][0] == NAME) { - var to = currAssign[2][1]; + Ref currAssign = curr[1]; + if (currAssign[1]->isBool(true) && currAssign[2][0] == NAME) { + IString to = currAssign[2][1]->getIString(); if (to == helper) { found = i; break; @@ -1446,17 +1485,17 @@ void eliminate(Ref ast, bool memSafe=false) { // first, see if the looper and helpers overlap. Note that we check for this looper, compared to // *ALL* the helpers. Helpers will be replaced by loopers as we eliminate them, potentially // causing conflicts, so any helper is a concern. - var firstLooperUsage = -1; - var lastLooperUsage = -1; - var firstHelperUsage = -1; - for (var i = found+1; i < stats.length; i++) { - var curr = i < stats.length-1 ? stats[i] : last[1]; // on the last line, just look in the condition - traverse(curr, function(node, type) { - if (type == NAME) { + int firstLooperUsage = -1; + int lastLooperUsage = -1; + int firstHelperUsage = -1; + for (int i = found+1; i < stats->size(); i++) { + Ref curr = i < stats->size()-1 ? stats[i] : last[1]; // on the last line, just look in the condition + traversePre(curr, [&](Ref node) { + if (node[0] == NAME) { if (node[1] == looper) { if (firstLooperUsage < 0) firstLooperUsage = i; lastLooperUsage = i; - } else if (helpers.indexOf(node[1]) >= 0) { + } else if (indexOf(helpers, node[1]->getIString()) >= 0) { if (firstHelperUsage < 0) firstHelperUsage = i; } } @@ -1464,72 +1503,73 @@ void eliminate(Ref ast, bool memSafe=false) { } if (firstLooperUsage >= 0) { // the looper is used, we cannot simply merge the two variables - if ((firstHelperUsage < 0 || firstHelperUsage > lastLooperUsage) && lastLooperUsage+1 < stats.length && triviallySafeToMove(stats[found], asmData) && + if ((firstHelperUsage < 0 || firstHelperUsage > lastLooperUsage) && lastLooperUsage+1 < stats->size() && triviallySafeToMove(stats[found], asmData) && seenUses[helper] == namings[helper]) { // the helper is not used, or it is used after the last use of the looper, so they do not overlap, // and the last looper usage is not on the last line (where we could not append after it), and the // helper is not used outside of the loop. // just move the looper definition to after the looper's last use - stats.splice(lastLooperUsage+1, 0, stats[found]); - stats.splice(found, 1); + stats->insert(lastLooperUsage+1, stats[found]); + stats->splice(found, 1); } else { // they overlap, we can still proceed with the loop optimization, but we must introduce a // loop temp helper variable - var temp = looper + '$looptemp'; - assert(!(temp in asmData.vars)); - for (var i = firstLooperUsage; i <= lastLooperUsage; i++) { - var curr = i < stats.length-1 ? stats[i] : last[1]; // on the last line, just look in the condition - traverse(curr, function looperToLooptemp(node, type) { - if (type == NAME) { + IString temp((std::string(looper.c_str()) + "$looptemp").c_str()); + assert(!asmData.isLocal(temp)); + for (int i = firstLooperUsage; i <= lastLooperUsage; i++) { + Ref curr = i < stats->size()-1 ? stats[i] : last[1]; // on the last line, just look in the condition + + std::function looperToLooptemp = [&](Ref node) { + if (node[0] == NAME) { if (node[1] == looper) { - node[1] = temp; + node[1]->setString(temp); } - } else if (type == ASSIGN && node[2][0] == NAME) { + } else if (node[0] == ASSIGN && node[2][0] == NAME) { // do not traverse the assignment target, phi assignments to the loop variable must remain - traverse(node[3], looperToLooptemp); - return null; + traversePrePostConditional(node[3], looperToLooptemp, [](Ref node){}); + return false; } - }); + return true; + }; + traversePrePostConditional(curr, looperToLooptemp, [](Ref node){}); } - asmData.vars[temp] = asmData.vars[looper]; - stats.splice(found, 0, [STAT, [ASSIGN, true, [NAME, temp], [NAME, looper]]]); + asmData.addVar(temp, asmData.getType(looper)); + stats->insert(found, make1(STAT, make3(ASSIGN, &(arena.alloc())->setBool(true), makeName(temp), makeName(looper)))); } } } - for (var l = 0; l < helpers.length; l++) { - for (var k = 0; k < helpers.length; k++) { + for (int l = 0; l < helpers.size(); l++) { + for (int k = 0; k < helpers.size(); k++) { if (l != k && helpers[l] == helpers[k]) return; // it is complicated to handle a shared helper, abort } } // hurrah! this is safe to do - //printErr("ELIM LOOP VAR " + JSON.stringify(loopers) + ' :: ' + JSON.stringify(helpers)); - for (var l = 0; l < loopers.length; l++) { - var looper = loopers[l]; - var helper = helpers[l]; + for (int l = 0; l < loopers.size(); l++) { + IString looper = loopers[l]; + IString helper = helpers[l]; varsToRemove[helper] = 2; - traverse(node, function(node, type) { // replace all appearances of helper with looper - if (type == NAME && node[1] == helper) node[1] = looper; + traversePre(node, [&](Ref node) { // replace all appearances of helper with looper + if (node[0] == NAME && node[1] == helper) node[1]->setString(looper); }); helperReplacements[helper] = looper; // replace all future appearances of helper with looper helperReplacements[looper] = looper; // avoid any further attempts to optimize looper in this manner (seenUses is wrong anyhow, too) } // simplify the if. we remove the if branch, leaving only the else if (flip) { - last[1] = simplifyNotCompsDirect([UNARY_PREFIX, '!', last[1]]); - var temp = last[2]; + last[1] = simplifyNotCompsDirect(make2(UNARY_PREFIX, L_NOT, last[1])); + Ref temp = last[2]; last[2] = last[3]; last[3] = temp; } - if (loopers.length == assigns.length) { - last.pop(); + if (loopers.size() == assigns->size()) { + last->pop_back(); } else { - var elseStats = getStatements(last[3]); - for (var i = 0; i < elseStats.length; i++) { - var stat = elseStats[i]; - if (stat[0] == STAT) stat = stat[1]; + Ref elseStats = getStatements(last[3]); + for (int i = 0; i < elseStats->size(); i++) { + Ref stat = deStat(elseStats[i]); if (stat[0] == ASSIGN && stat[2][0] == NAME) { - if (loopers.indexOf(stat[2][1]) >= 0) { - elseStats[i] = emptyNode(); + if (indexOf(loopers, stat[2][1]->getIString()) >= 0) { + elseStats[i] = makeEmpty(); } } } @@ -1537,7 +1577,7 @@ void eliminate(Ref ast, bool memSafe=false) { } } } - }); */ + }); for (auto v : varsToRemove) { if (v.second == 2) asmData.deleteVar(v.first); From 05faca9b342a35d125433947de2ed6bd9f973bd5 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Nov 2014 17:06:43 -0800 Subject: [PATCH 103/161] eliminate fixes --- tools/optimizer/minijson.h | 1 + tools/optimizer/optimizer.cpp | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 0c614842b4157..960385b50b18b 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -21,6 +21,7 @@ #define err(str) fprintf(stderr, str "\n"); #define errv(str, ...) fprintf(stderr, str "\n", __VA_ARGS__); +#define printErr err struct IString; class Ref; diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index c9ec4ed176796..5b7138322c788 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -88,6 +88,12 @@ int jsD2I(double x) { return (int)((int64_t)x); } +char *strdupe(const char *str) { + char *ret = (char *)malloc(strlen(str)+1); // leaked! + strcpy(ret, str); + return ret; +} + int parseInt(const char *str) { int ret = *str - '0'; while (*(++str)) { @@ -1390,11 +1396,7 @@ void eliminate(Ref ast, bool memSafe=false) { return; // no need to track this anymore, we can't loop-optimize more than once } // track how many uses we saw. we need to know when a variable is no longer used (hence we run this in the post) - if (seenUses.has(name)) { - seenUses[name] = 1; - } else { - seenUses[name]++; - } + seenUses[name]++; } else if (type == WHILE) { // try to remove loop helper variables specifically Ref stats = node[2][1]; @@ -1514,7 +1516,7 @@ void eliminate(Ref ast, bool memSafe=false) { } else { // they overlap, we can still proceed with the loop optimization, but we must introduce a // loop temp helper variable - IString temp((std::string(looper.c_str()) + "$looptemp").c_str()); + IString temp(strdupe((std::string(looper.c_str()) + "$looptemp").c_str())); assert(!asmData.isLocal(temp)); for (int i = firstLooperUsage; i <= lastLooperUsage; i++) { Ref curr = i < stats->size()-1 ? stats[i] : last[1]; // on the last line, just look in the condition From abb4573d3d3d7f13092afb1af3754d70b847a26b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Nov 2014 17:22:39 -0800 Subject: [PATCH 104/161] finish eliminator port --- tools/optimizer/optimizer.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 5b7138322c788..29df94572663f 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -1047,6 +1047,7 @@ void eliminate(Ref ast, bool memSafe=false) { HASES }; Tracked tracked; + #define dumpTracked() { errv("tracking %d", tracked.size()); for (auto t : tracked) errv("... %s", t.first.c_str()); } bool globalsInvalidated = false; // do not repeat invalidations, until we track something new bool memoryInvalidated = false; @@ -1308,7 +1309,18 @@ void eliminate(Ref ast, bool memSafe=false) { traverseInOrder(stats[j], false, false); } // We cannot track from one switch case into another, undo all new trackings TODO: general framework here, use in if-else as well - tracked = originalTracked; // TODO: don't do this on the last one + std::vector toDelete; + for (auto t : tracked) { + if (!originalTracked.has(t.first)) { + Tracking& info = tracked[t.first]; + if (info.usesGlobals || info.usesMemory || info.deps.size() > 0) { + toDelete.push_back(t.first); + } + } + } + for (auto t : toDelete) { + tracked.erase(t); + } } tracked.clear(); // do not track from inside the switch to outside } else { From 0b54b3d3f20ae84f1ed7d140da5a42af728ebb25 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Nov 2014 17:23:14 -0800 Subject: [PATCH 105/161] enable eliminate and eliminateMemSafe passes in native optimizer --- tools/js_optimizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 9c1e64c5a4525..6f912a34d8083 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -9,7 +9,7 @@ def path_from_root(*pathelems): return os.path.join(__rootpath__, *pathelems) -NATIVE_PASSES = set(['asm', 'asmPreciseF32', 'receiveJSON', 'emitJSON', 'simplifyExpressions', 'simplifyIfs', 'optimizeFrounds']) +NATIVE_PASSES = set(['asm', 'asmPreciseF32', 'receiveJSON', 'emitJSON', 'eliminate', 'eliminateMemSafe', 'simplifyExpressions', 'simplifyIfs', 'optimizeFrounds']) JS_OPTIMIZER = path_from_root('tools', 'js-optimizer.js') From a6af6aa258860047b0e5dfbb9df130df15b9dcbd Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Nov 2014 17:53:45 -0800 Subject: [PATCH 106/161] fix bug with removing a param as if it were a var --- tools/optimizer/optimizer.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 29df94572663f..295704f94952d 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -265,6 +265,9 @@ struct AsmData { bool isLocal(const IString& name) { return locals.count(name) > 0; } + bool isVar(const IString& name) { + return isLocal(name) && !locals[name].param; + } AsmData(Ref f) { func = f; @@ -358,6 +361,7 @@ struct AsmData { int next = 0; for (auto param : func[2]->getArray()) { IString str = param->getIString(); + assert(locals.count(str) > 0); stats[next++] = make1(STAT, make3(ASSIGN, &(arena.alloc())->setBool(true), makeName(str.c_str()), makeAsmCoercion(makeName(str.c_str()), locals[str].type))); } if (varDefs->size()) { @@ -1594,7 +1598,7 @@ void eliminate(Ref ast, bool memSafe=false) { }); for (auto v : varsToRemove) { - if (v.second == 2) asmData.deleteVar(v.first); + if (v.second == 2 && asmData.isVar(v.first)) asmData.deleteVar(v.first); } asmData.denormalize(); From 6c5eae533ab9946ac5e77acaa15732da9cdb0a97 Mon Sep 17 00:00:00 2001 From: Chad Austin Date: Tue, 11 Nov 2014 22:26:29 -0800 Subject: [PATCH 107/161] fix upstream embind tests --- tests/embind/embind.test.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/embind/embind.test.js b/tests/embind/embind.test.js index bb373eaea5971..e26822468ccd5 100644 --- a/tests/embind/embind.test.js +++ b/tests/embind/embind.test.js @@ -476,7 +476,7 @@ module({ assert.equal(true, cm.emval_test_not(false)); }); - test("val.is_undefined() is functionnal",function() { + test("val.is_undefined() is functional",function() { assert.equal(true, cm.emval_test_is_undefined(undefined)); assert.equal(false, cm.emval_test_is_undefined(true)); assert.equal(false, cm.emval_test_is_undefined(false)); @@ -484,7 +484,7 @@ module({ assert.equal(false, cm.emval_test_is_undefined({})); }); - test("val.is_null() is functionnal",function() { + test("val.is_null() is functional",function() { assert.equal(true, cm.emval_test_is_null(null)); assert.equal(false, cm.emval_test_is_null(true)); assert.equal(false, cm.emval_test_is_null(false)); @@ -492,7 +492,7 @@ module({ assert.equal(false, cm.emval_test_is_null({})); }); - test("val.is_true() is functionnal",function() { + test("val.is_true() is functional",function() { assert.equal(true, cm.emval_test_is_true(true)); assert.equal(false, cm.emval_test_is_true(false)); assert.equal(false, cm.emval_test_is_true(null)); @@ -500,7 +500,7 @@ module({ assert.equal(false, cm.emval_test_is_true({})); }); - test("val.is_false() is functionnal",function() { + test("val.is_false() is functional",function() { assert.equal(true, cm.emval_test_is_false(false)); assert.equal(false, cm.emval_test_is_false(true)); assert.equal(false, cm.emval_test_is_false(null)); @@ -508,8 +508,8 @@ module({ assert.equal(false, cm.emval_test_is_false({})); }); - test("val.equals() is functionnal",function() { - values = [undefined, null, true, false, {}]; + test("val.equals() is functional",function() { + var values = [undefined, null, true, false, {}]; for(var i=0;i Date: Tue, 11 Nov 2014 22:28:29 -0800 Subject: [PATCH 108/161] jshint --- tests/embind/embind.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/embind/embind.test.js b/tests/embind/embind.test.js index e26822468ccd5..1f5a51076d526 100644 --- a/tests/embind/embind.test.js +++ b/tests/embind/embind.test.js @@ -516,8 +516,8 @@ module({ for(var j=i;j Date: Mon, 30 Jun 2014 13:18:57 -0700 Subject: [PATCH 109/161] some minor fixes that got in the way of debugging a separate issue --- tests/embind/embind_test.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/embind/embind_test.cpp b/tests/embind/embind_test.cpp index b8cf90b79a275..b6cc245bb9d6a 100644 --- a/tests/embind/embind_test.cpp +++ b/tests/embind/embind_test.cpp @@ -1290,10 +1290,10 @@ EMSCRIPTEN_BINDINGS(interface_tests) { .smart_ptr>("shared_ptr") .allow_subclass("AbstractClassWrapper") .function("abstractMethod", &AbstractClass::abstractMethod, pure_virtual()) - // The select_overload is necessary because, otherwise, the C++ compiler + // The optional_override is necessary because, otherwise, the C++ compiler // cannot deduce the signature of the lambda function. .function("optionalMethod", optional_override( - [](AbstractClass& this_, std::string s) { + [](AbstractClass& this_, const std::string& s) { return this_.AbstractClass::optionalMethod(s); } )) @@ -2138,6 +2138,8 @@ class MultipleOverloads { } }; +int MultipleOverloads::staticValue = 0; + class MultipleOverloadsDerived : public MultipleOverloads { public: MultipleOverloadsDerived() {} From 0555b29d2ac4d312304499bae38eaa4df7e1b863 Mon Sep 17 00:00:00 2001 From: Chad Austin Date: Fri, 30 Aug 2013 18:25:52 -0700 Subject: [PATCH 110/161] IE11 compatibility --- src/embind/embind.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/embind/embind.js b/src/embind/embind.js index 6f0f8c13a8d6a..36486e6009a7a 100644 --- a/src/embind/embind.js +++ b/src/embind/embind.js @@ -769,7 +769,7 @@ var LibraryEmbind = { * isn't very helpful. Using IMVU.createNamedFunction addresses the issue. Doublely-unfortunately, there's no way * to write a test for this behavior. -NRD 2013.02.22 */ - var dummy = createNamedFunction(constructor.name, function(){}); + var dummy = createNamedFunction(constructor.name || 'unknownFunctionName', function(){}); dummy.prototype = constructor.prototype; var obj = new dummy; From 69529d09c2a995b2a870a415ec68371fa03f6f9b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Nov 2014 11:23:53 -0800 Subject: [PATCH 111/161] no need for exceptions or rtti in cpp optimizer --- tools/js_optimizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 6f912a34d8083..db7228efedd3b 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -29,7 +29,7 @@ def create_optimizer(): shared.logging.debug('building native optimizer') output = shared.Cache.get_path('optimizer.exe') shared.try_delete(output) - subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-O3', '-std=c++11', '-o', output]).communicate() # , '-g', '-fno-omit-frame-pointer' + subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-O3', '-std=c++11', '-fno-exceptions', '-fno-rtti', '-o', output]).communicate() # , '-g', '-fno-omit-frame-pointer' assert os.path.exists(output) return output return shared.Cache.get('optimizer.exe', create_optimizer, extension='exe') From e75930d8aa92d0b798e6f9290e525053b58be72c Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Nov 2014 11:24:12 -0800 Subject: [PATCH 112/161] little optimization on traversePre --- tools/optimizer/optimizer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 295704f94952d..fe7bb962ab4d3 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -127,7 +127,7 @@ void traversePre(Ref node, std::function visit) { visit(node); std::vector stack; stack.push_back({ node, 0 }); - while (stack.size() > 0) { + while (1) { TraverseInfo& top = stack.back(); if (top.index < top.node->size()) { Ref sub = top.node[top.index]; @@ -137,6 +137,7 @@ void traversePre(Ref node, std::function visit) { stack.push_back({ sub, 0 }); } } else { + if (stack.size() == 1) break; stack.pop_back(); } } From eb8b83c5830e982bc2218c4e9c128b19221e74b1 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Nov 2014 13:00:33 -0800 Subject: [PATCH 113/161] optimize traversePre with a stack-based stack --- tools/optimizer/optimizer.cpp | 49 ++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index fe7bb962ab4d3..9ab11fde4a932 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -120,12 +120,59 @@ struct TraverseInfo { int index; }; +template +struct StackedStack { // a stack, on the stack + T stackStorage[init]; + T* storage; + int used, available; // used amount, available amount + bool alloced; + + StackedStack() : used(0), available(init), alloced(false) { + storage = stackStorage; + } + ~StackedStack() { + if (alloced) free(storage); + } + + int size() { return used; } + + void push_back(const T& t) { + assert(used <= available); + if (used == available) { + available *= 2; + if (!alloced) { + T* old = storage; + storage = (T*)malloc(sizeof(T)*available); + memcpy(storage, old, sizeof(T)*used); + alloced = true; + } else { + storage = (T*)realloc(storage, sizeof(T)*available); + } + } + assert(used < available); + assert(storage); + storage[used++] = t; + } + + T& back() { + assert(used > 0); + return storage[used-1]; + } + + void pop_back() { + assert(used > 0); + used--; + } +}; + #define visitable(node) (node->isArray() and node->size() > 0) +#define TRAV_STACK 40 + // Traverse, calling visit before the children void traversePre(Ref node, std::function visit) { visit(node); - std::vector stack; + StackedStack stack; stack.push_back({ node, 0 }); while (1) { TraverseInfo& top = stack.back(); From 68b550a8d1f204fc72a0f19d3fccf53da0e024f2 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Nov 2014 13:06:47 -0800 Subject: [PATCH 114/161] additional small traversal opts --- tools/optimizer/optimizer.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 9ab11fde4a932..e874a281bf3e5 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -174,7 +174,7 @@ void traversePre(Ref node, std::function visit) { visit(node); StackedStack stack; stack.push_back({ node, 0 }); - while (1) { + while (stack.size() > 0) { TraverseInfo& top = stack.back(); if (top.index < top.node->size()) { Ref sub = top.node[top.index]; @@ -184,7 +184,6 @@ void traversePre(Ref node, std::function visit) { stack.push_back({ sub, 0 }); } } else { - if (stack.size() == 1) break; stack.pop_back(); } } @@ -193,7 +192,7 @@ void traversePre(Ref node, std::function visit) { // Traverse, calling visitPre before the children and visitPost after void traversePrePost(Ref node, std::function visitPre, std::function visitPost) { visitPre(node); - std::vector stack; + StackedStack stack; stack.push_back({ node, 0 }); while (stack.size() > 0) { TraverseInfo& top = stack.back(); @@ -215,7 +214,7 @@ void traversePrePost(Ref node, std::function visitPre, std::function // Traverse, calling visitPre before the children and visitPost after. If pre returns false, do not traverse children void traversePrePostConditional(Ref node, std::function visitPre, std::function visitPost) { if (!visitPre(node)) return; - std::vector stack; + StackedStack stack; stack.push_back({ node, 0 }); while (stack.size() > 0) { TraverseInfo& top = stack.back(); From f418b5f5504805095915dab72d39bcebad5c32c1 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Nov 2014 14:47:33 -0800 Subject: [PATCH 115/161] initial work to port registerize --- tools/optimizer/minijson.h | 14 ++ tools/optimizer/optimizer.cpp | 281 +++++++++++++++++++++++++++++++++- 2 files changed, 291 insertions(+), 4 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 960385b50b18b..254afa2fa9589 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -121,6 +121,10 @@ struct IString { return str[x]; } + bool operator!() { // no string, or empty string + return !str || str[0] == 0; + } + const char *c_str() const { return str; } bool isNull() { return str == nullptr; } @@ -467,6 +471,16 @@ struct Value { return -1; } + Ref map(std::function func) { + assert(isArray()); + Ref ret = arena.alloc(); + ret->setArray(); + for (unsigned i = 0; i < arr->size(); i++) { + ret->push_back(func((*arr)[i])); + } + return ret; + } + /* void forEach(std::function func) { for (unsigned i = 0; i < arr->size(); i++) { diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index e874a281bf3e5..19f0aeec8268e 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -25,6 +25,7 @@ IString TOPLEVEL("toplevel"), IF("if"), WHILE("while"), DO("do"), + FOR("for"), SEQ("seq"), SUB("sub"), CALL("call"), @@ -270,7 +271,7 @@ enum AsmType { ASM_FLOAT = 2, ASM_FLOAT32X4 = 3, ASM_INT32X4 = 4, - ASM_NONE = 5 + ASM_NONE = 5 // number of types }; // forward decls @@ -312,6 +313,9 @@ struct AsmData { bool isLocal(const IString& name) { return locals.count(name) > 0; } + bool isParam(const IString& name) { + return isLocal(name) && locals[name].param; + } bool isVar(const IString& name) { return isLocal(name) && !locals[name].param; } @@ -415,7 +419,7 @@ struct AsmData { stats[next] = make1(VAR, varDefs); } /* - if (inlines.length > 0) { + if (inlines->size() > 0) { var i = 0; traverse(func, function(node, type) { if (type == CALL && node[1][0] == NAME && node[1][1] == 'inlinejs') { @@ -438,6 +442,10 @@ struct AsmData { //printErr('denormalized \n\n' + astToSrc(func) + '\n\n'); } + void addParam(IString name, AsmType type) { + locals[name] = { type, true }; + params.push_back(name); + } void addVar(IString name, AsmType type) { locals[name] = { type, false }; vars.push_back(name); @@ -835,6 +843,24 @@ void removeAllEmptySubNodes(Ref ast) { }); } +Ref unVarify(Ref vars) { // transform var x=1, y=2 etc. into (x=1, y=2), i.e., the same assigns, but without a var definition + Ref ret; + ret->push_back(makeString(STAT)); + if (vars->size() == 1) { + ret->push_back(make3(ASSIGN, &(arena.alloc())->setBool(true), makeName(vars[0][0]->getIString()), vars[0][1])); + } else { + ret->push_back(makeArray()); + Ref curr = ret[1]; + for (int i = 0; i < vars->size()-1; i++) { + curr->push_back(makeString(SEQ)); + curr->push_back(make3(ASSIGN, &(arena.alloc())->setBool(true), makeName(vars[i][0]->getIString()), vars[i][1])); + if (i != vars->size()-2) curr = curr[2] = makeArray(); + } + curr[2] = make3(ASSIGN, &(arena.alloc())->setBool(true), makeName(vars[vars->size()-1][0]->getIString()), vars[vars->size()-1][1]); + } + return ret; +} + // Calculations int measureCost(Ref ast) { @@ -898,6 +924,8 @@ StringSet USEFUL_BINARY_OPS("<< >> | & ^"), COERCION_REQUIRING_BINARIES("* / %"); // binary ops that in asm must be coerced StringSet ASSOCIATIVE_BINARIES("+ * | & ^"), + CONTROL_FLOW("do while for if switch"), + LOOP("do while for"), NAME_OR_NUM("name num"); bool isFunctionTable(const char *name) { @@ -970,6 +998,11 @@ class StringRefMap : public std::unordered_map { HASES }; +class StringTypeMap : public std::unordered_map { +public: + HASES +}; + void eliminate(Ref ast, bool memSafe=false) { // Find variables that have a single use, and if they can be eliminated, do so traverseFunctions(ast, [&memSafe](Ref func) { @@ -1437,9 +1470,9 @@ void eliminate(Ref ast, bool memSafe=false) { traversePrePost(func, [&](Ref node) { // pre Ref type = node[0]; - /*if (type === VAR) { + /*if (type == VAR) { node[1] = node[1].filter(function(pair) { return !varsToRemove[pair[0]] }); - if (node[1].length === 0) { + if (node[1]->size() == 0) { // wipe out an empty |var;| node[0] = 'toplevel'; node[1] = []; @@ -2278,6 +2311,245 @@ void optimizeFrounds(Ref ast) { }); } +// Very simple 'registerization', coalescing of variables into a smaller number. +void registerize(Ref ast) { + traverseFunctions(ast, [](Ref fun) { + AsmData asmData(fun); + // Add parameters as a first (fake) var (with assignment), so they get taken into consideration + // note: params are special, they can never share a register between them (see later) + if (!!fun[2] && fun[2]->size()) { + Ref assign = makeNum(0); + fun[3]->insert(0, make1(VAR, fun[2]->map([&assign](Ref param) { + return &(makeArray()->push_back(param).push_back(assign)); + }))); + } + // Replace all var definitions with assignments; we will add var definitions at the top after we registerize + StringSet allVars; + traversePre(fun, [&](Ref node) { + Ref type = node[0]; + if (type == VAR) { + Ref vars = node[1]; // XXX.filter(function(varr) { return varr[1] }); + if (vars->size() > 0) { + safeCopy(node, unVarify(vars)); + } else { + safeCopy(node, makeEmpty()); + } + } else if (type == NAME) { + allVars.insert(node[1]->getIString()); + } + }); + removeAllEmptySubNodes(fun); // vacuum? + StringTypeMap regTypes; // reg name -> type + auto getNewRegName = [&](int num, IString name) { + std::string str; + AsmType type = asmData.getType(name); + switch (type) { + case ASM_INT: str = "i"; break; + case ASM_DOUBLE: str = "d"; break; + case ASM_FLOAT: str = "f"; break; + case ASM_FLOAT32X4: str = "F4"; break; + case ASM_INT32X4: str = "I4"; break; + case ASM_NONE: str = "Z"; break; + default: assert(0); // type doesn't have a name yet + } + str += num; + IString ret(strdupe(str.c_str())); // likely interns a new string; leaks the dupe if not + regTypes[ret] = type; + assert(!allVars.has(ret)); // register must not shadow non-local name + return ret; + }; + // Find the # of uses of each variable. + // While doing so, check if all a variable's uses are dominated in a simple + // way by a simple assign, if so, then we can assign its register to it + // just for its definition to its last use, and not to the entire toplevel loop, + // we call such variables "optimizable" + StringIntMap varUses; + int level = 1; + std::unordered_map levelDominations; // level => set of dominated variables XXX vector? + StringIntMap varLevels; + StringSet possibles; + StringSet unoptimizables; + auto purgeLevel = [&]() { + // Invalidate all dominating on this level, further users make it unoptimizable + for (auto name : levelDominations[level]) { + varLevels[name] = 0; + } + levelDominations[level].clear(); + level--; + }; + std::function possibilifier = [&](Ref node) { + Ref type = node[0]; + if (type == NAME) { + IString name = node[1]->getIString(); + if (asmData.isLocal(name)) { + varUses[name]++; + if (possibles.has(name) && !varLevels[name]) unoptimizables.insert(name); // used outside of simple domination + } + } else if (type == ASSIGN && node[1]->isBool(true)) { + if (!!node[2] && node[2][0] == NAME) { + IString name = node[2][1]->getIString(); + // if local and not yet used, this might be optimizable if we dominate + // all other uses + if (asmData.isLocal(name) && !varUses[name] && !varLevels[name]) { + possibles.insert(name); + varLevels[name] = level; + levelDominations[level].insert(name); + } + } + } else if (CONTROL_FLOW.has(type)) { + // recurse children, in the context of a loop + if (type == WHILE || type == DO) { + traversePrePostConditional(node[1], possibilifier, [](Ref node){}); + level++; + traversePrePostConditional(node[2], possibilifier, [](Ref node){}); + purgeLevel(); + } else if (type == FOR) { + traversePrePostConditional(node[1], possibilifier, [](Ref node){}); + for (int i = 2; i <= 4; i++) { + level++; + traversePrePostConditional(node[i], possibilifier, [](Ref node){}); + purgeLevel(); + } + } else if (type == IF) { + traversePrePostConditional(node[1], possibilifier, [](Ref node){}); + level++; + traversePrePostConditional(node[2], possibilifier, [](Ref node){}); + purgeLevel(); + if (!!node[3]) { + level++; + traversePrePostConditional(node[3], possibilifier, [](Ref node){}); + purgeLevel(); + } + } else if (type == SWITCH) { + traversePrePostConditional(node[1], possibilifier, [](Ref node){}); + Ref cases = node[2]; + for (int i = 0; i < cases->size(); i++) { + level++; + traversePrePostConditional(cases[i][1], possibilifier, [](Ref node){}); + purgeLevel(); + } + } else assert(0);; + return false; // prevent recursion into children, which we already did + } + return true; + }; + traversePrePostConditional(fun, possibilifier, [](Ref node){}); + + StringSet optimizables; + for (auto possible : possibles) { + if (!unoptimizables.has(possible)) optimizables.insert(possible); + } + + // Go through the function's code, assigning 'registers'. + // The only tricky bit is to keep variables locked on a register through loops, + // since they can potentially be returned to. Optimizable variables lock onto + // loops that they enter, unoptimizable variables lock in a conservative way + // into the topmost loop. + // Note that we cannot lock onto a variable in a loop if it was used and free'd + // before! (then they could overwrite us in the early part of the loop). For now + // we just use a fresh register to make sure we avoid this, but it could be + // optimized to check for safe registers (free, and not used in this loop level). + StringStringMap varRegs; // maps variables to the register they will use all their life + typedef std::vector StringVec; + std::vector freeRegsClasses; + freeRegsClasses.resize(ASM_NONE); + int nextReg = 1; + StringVec fullNames; + fullNames.push_back(EMPTY); // names start at 1 + std::vector loopRegs; // for each loop nesting level, the list of bound variables + int loops = 0; // 0 is toplevel, 1 is first loop, etc + StringSet activeOptimizables; + StringIntMap optimizableLoops; + StringSet paramRegs; // true if the register is used by a parameter (and so needs no def at start of function; also cannot + // be shared with another param, each needs its own) + auto decUse = [&](IString name) { + if (!varUses[name]) return false; // no uses left, or not a relevant variable + if (optimizables.has(name)) activeOptimizables.insert(name); + IString reg = varRegs[name]; + assert(asmData.isLocal(name)); + StringVec& freeRegs = freeRegsClasses[asmData.getType(name)]; + if (!reg) { + // acquire register + if (optimizables.has(name) && freeRegs.size() > 0 && + !(asmData.isParam(name) && paramRegs.has(freeRegs.back()))) { // do not share registers between parameters + reg = freeRegs.back(); + freeRegs.pop_back(); + } else { + fullNames[nextReg] = reg = getNewRegName(nextReg, name); + if (asmData.isParam(name)) paramRegs.insert(reg); + nextReg++; + } + varRegs[name] = reg; + } + varUses[name]--; + assert(varUses[name] >= 0); + if (varUses[name] == 0) { + if (optimizables.has(name)) activeOptimizables.erase(name); + // If we are not in a loop, or we are optimizable and not bound to a loop + // (we might have been in one but left it), we can free the register now. + if (loops == 0 || (optimizables.has(name) && !optimizableLoops.has(name))) { + // free register + freeRegs.push_back(reg); + } else { + // when the relevant loop is exited, we will free the register + int relevantLoop = optimizables.has(name) ? (optimizableLoops[name] ? optimizableLoops[name] : 1) : 1; + if (loopRegs.size() <= relevantLoop) loopRegs.resize(relevantLoop); + loopRegs[relevantLoop].push_back(reg); + } + } + return true; + }; + traversePrePost(fun, [&](Ref node) { // XXX we rely on traversal order being the same as execution order here + Ref type = node[0]; + if (type == NAME) { + IString name = node[1]->getIString(); + if (decUse(name)) { + node[1]->setString(varRegs[name]); + } + } else if (LOOP.has(type)) { + loops++; + // Active optimizables lock onto this loop, if not locked onto one that encloses this one + for (auto name : activeOptimizables) { + if (!optimizableLoops[name]) { + optimizableLoops[name] = loops; + } + } + } + }, [&](Ref node) { + Ref type = node[0]; + if (LOOP.has(type)) { + // Free registers that were locked to this loop + if (loopRegs[loops].size() > 0) { + for (auto loopReg : loopRegs[loops]) { + freeRegsClasses[regTypes[loopReg]].push_back(loopReg); + } + loopRegs[loops].clear(); + } + loops--; + } + }); + if (!!fun[2] && fun[2]->size()) { + fun[2]->setSize(0); // clear params, we will fill with registers + fun[3]->splice(0, 1); // remove fake initial var + } + + asmData.locals.clear(); + asmData.params.clear(); + asmData.vars.clear(); + for (int i = 1; i < nextReg; i++) { + IString reg = fullNames[i]; + AsmType type = regTypes[reg]; + if (!paramRegs.has(reg)) { + asmData.addVar(reg, type); + } else { + asmData.addParam(reg, type); + fun[2]->push_back(makeString(reg)); + } + } + asmData.denormalize(); + }); +} + //================== // Main //================== @@ -2318,6 +2590,7 @@ int main(int argc, char **argv) { else if (str == "simplifyExpressions") simplifyExpressions(doc); else if (str == "optimizeFrounds") optimizeFrounds(doc); else if (str == "simplifyIfs") simplifyIfs(doc); + else if (str == "registerize") registerize(doc); else { fprintf(stderr, "unrecognized argument: %s\n", str.c_str()); assert(0); From aff6780ade1137b657887d330b2fa9a369c369ec Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Nov 2014 16:15:35 -0800 Subject: [PATCH 116/161] fix some registerize crashes --- tools/optimizer/optimizer.cpp | 38 ++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 19f0aeec8268e..18e4963727e79 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -844,7 +844,7 @@ void removeAllEmptySubNodes(Ref ast) { } Ref unVarify(Ref vars) { // transform var x=1, y=2 etc. into (x=1, y=2), i.e., the same assigns, but without a var definition - Ref ret; + Ref ret = makeArray(); ret->push_back(makeString(STAT)); if (vars->size() == 1) { ret->push_back(make3(ASSIGN, &(arena.alloc())->setBool(true), makeName(vars[0][0]->getIString()), vars[0][1])); @@ -894,7 +894,7 @@ bool preciseF32 = false; return count(str) > 0; \ } \ bool has(Ref node) { \ - return count(node->getIString()) > 0; \ + return node->isString() && count(node->getIString()) > 0; \ } class StringSet : public std::unordered_set { @@ -914,6 +914,14 @@ class StringSet : public std::unordered_set { } HASES + + void dump() { + err("==="); + for (auto str : *this) { + errv("%s", str.c_str()); + } + err("==="); + } }; StringSet USEFUL_BINARY_OPS("<< >> | & ^"), @@ -2320,7 +2328,7 @@ void registerize(Ref ast) { if (!!fun[2] && fun[2]->size()) { Ref assign = makeNum(0); fun[3]->insert(0, make1(VAR, fun[2]->map([&assign](Ref param) { - return &(makeArray()->push_back(param).push_back(assign)); + return &(makeArray()->push_back(param)); // push_back(assign) }))); } // Replace all var definitions with assignments; we will add var definitions at the top after we registerize @@ -2329,11 +2337,8 @@ void registerize(Ref ast) { Ref type = node[0]; if (type == VAR) { Ref vars = node[1]; // XXX.filter(function(varr) { return varr[1] }); - if (vars->size() > 0) { - safeCopy(node, unVarify(vars)); - } else { - safeCopy(node, makeEmpty()); - } + for (int i = 0; i < vars->size(); i++) assert(vars[i]->size() == 1); + safeCopy(node, makeEmpty()); } else if (type == NAME) { allVars.insert(node[1]->getIString()); } @@ -2341,7 +2346,7 @@ void registerize(Ref ast) { removeAllEmptySubNodes(fun); // vacuum? StringTypeMap regTypes; // reg name -> type auto getNewRegName = [&](int num, IString name) { - std::string str; + const char *str; AsmType type = asmData.getType(name); switch (type) { case ASM_INT: str = "i"; break; @@ -2352,8 +2357,12 @@ void registerize(Ref ast) { case ASM_NONE: str = "Z"; break; default: assert(0); // type doesn't have a name yet } - str += num; - IString ret(strdupe(str.c_str())); // likely interns a new string; leaks the dupe if not + int size = strlen(str) + int(ceil(log10(num))) + 3; + char *temp = (char*)malloc(size); + int written = sprintf(temp, "%s%d", str, num); + assert(written < size); + temp[written] = 0; + IString ret(temp); // likely interns a new string; leaks if not XXX FIXME regTypes[ret] = type; assert(!allVars.has(ret)); // register must not shadow non-local name return ret; @@ -2475,9 +2484,10 @@ void registerize(Ref ast) { reg = freeRegs.back(); freeRegs.pop_back(); } else { - fullNames[nextReg] = reg = getNewRegName(nextReg, name); + assert(fullNames.size() == nextReg); + reg = getNewRegName(nextReg++, name); + fullNames.push_back(reg); if (asmData.isParam(name)) paramRegs.insert(reg); - nextReg++; } varRegs[name] = reg; } @@ -2493,7 +2503,7 @@ void registerize(Ref ast) { } else { // when the relevant loop is exited, we will free the register int relevantLoop = optimizables.has(name) ? (optimizableLoops[name] ? optimizableLoops[name] : 1) : 1; - if (loopRegs.size() <= relevantLoop) loopRegs.resize(relevantLoop); + if (loopRegs.size() <= relevantLoop+1) loopRegs.resize(relevantLoop+1); loopRegs[relevantLoop].push_back(reg); } } From 91a0d820c02a47d6e3976c82c6c4acc7b86aebdd Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Nov 2014 16:48:04 -0800 Subject: [PATCH 117/161] fix some registerize porting bugs --- tools/optimizer/minijson.h | 11 +++++++++++ tools/optimizer/optimizer.cpp | 15 ++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 254afa2fa9589..d853a9e279a20 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -481,6 +481,17 @@ struct Value { return ret; } + Ref filter(std::function func) { + assert(isArray()); + Ref ret = arena.alloc(); + ret->setArray(); + for (unsigned i = 0; i < arr->size(); i++) { + Ref curr = (*arr)[i]; + if (func(curr)) ret->push_back(curr); + } + return ret; + } + /* void forEach(std::function func) { for (unsigned i = 0; i < arr->size(); i++) { diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 18e4963727e79..ec73185be86fd 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -856,7 +856,7 @@ Ref unVarify(Ref vars) { // transform var x=1, y=2 etc. into (x=1, y=2), i.e., t curr->push_back(make3(ASSIGN, &(arena.alloc())->setBool(true), makeName(vars[i][0]->getIString()), vars[i][1])); if (i != vars->size()-2) curr = curr[2] = makeArray(); } - curr[2] = make3(ASSIGN, &(arena.alloc())->setBool(true), makeName(vars[vars->size()-1][0]->getIString()), vars[vars->size()-1][1]); + curr->push_back(make3(ASSIGN, &(arena.alloc())->setBool(true), makeName(vars[vars->size()-1][0]->getIString()), vars[vars->size()-1][1])); } return ret; } @@ -2325,10 +2325,12 @@ void registerize(Ref ast) { AsmData asmData(fun); // Add parameters as a first (fake) var (with assignment), so they get taken into consideration // note: params are special, they can never share a register between them (see later) + Ref fake; if (!!fun[2] && fun[2]->size()) { Ref assign = makeNum(0); + // TODO: will be an isEmpty here, can reuse it. fun[3]->insert(0, make1(VAR, fun[2]->map([&assign](Ref param) { - return &(makeArray()->push_back(param)); // push_back(assign) + return &(makeArray()->push_back(param).push_back(assign)); }))); } // Replace all var definitions with assignments; we will add var definitions at the top after we registerize @@ -2336,9 +2338,12 @@ void registerize(Ref ast) { traversePre(fun, [&](Ref node) { Ref type = node[0]; if (type == VAR) { - Ref vars = node[1]; // XXX.filter(function(varr) { return varr[1] }); - for (int i = 0; i < vars->size(); i++) assert(vars[i]->size() == 1); - safeCopy(node, makeEmpty()); + Ref vars = node[1]->filter([](Ref varr) { return varr->size() > 1; }); + if (vars->size() >= 1) { + safeCopy(node, unVarify(vars)); + } else { + safeCopy(node, makeEmpty()); + } } else if (type == NAME) { allVars.insert(node[1]->getIString()); } From 1a58369e2f3b3792b48452d44332e95c41b2cde4 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Nov 2014 16:55:59 -0800 Subject: [PATCH 118/161] print passes in js optimizer test --- tests/test_other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_other.py b/tests/test_other.py index c99dd14c3ab12..94d884aff7a5c 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -1933,7 +1933,7 @@ def test_js_optimizer(self): (path_from_root('tools', 'test-js-optimizer-ensureLabelSet.js'), open(path_from_root('tools', 'test-js-optimizer-ensureLabelSet-output.js')).read(), ['asm', 'ensureLabelSet']), ]: - print input + print input, passes # test calling js optimizer print ' js' output = Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), input] + passes, stdin=PIPE, stdout=PIPE).communicate()[0] From fd738c6754f5b7e353d3212ba1f94d760e1109a7 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Nov 2014 17:02:50 -0800 Subject: [PATCH 119/161] remove side-effect-free nodes in registerize; enable registerize in native optimizer --- tools/js_optimizer.py | 2 +- tools/optimizer/optimizer.cpp | 27 ++++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index db7228efedd3b..8252a8387f484 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -9,7 +9,7 @@ def path_from_root(*pathelems): return os.path.join(__rootpath__, *pathelems) -NATIVE_PASSES = set(['asm', 'asmPreciseF32', 'receiveJSON', 'emitJSON', 'eliminate', 'eliminateMemSafe', 'simplifyExpressions', 'simplifyIfs', 'optimizeFrounds']) +NATIVE_PASSES = set(['asm', 'asmPreciseF32', 'receiveJSON', 'emitJSON', 'eliminate', 'eliminateMemSafe', 'simplifyExpressions', 'simplifyIfs', 'optimizeFrounds', 'registerize']) JS_OPTIMIZER = path_from_root('tools', 'js-optimizer.js') diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index ec73185be86fd..da60d02a44fca 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -829,6 +829,19 @@ void clearEmptyNodes(Ref arr) { } if (skip) arr->setSize(arr->size() - skip); } +void clearUselessNodes(Ref arr) { + int skip = 0; + for (int i = 0; i < arr->size(); i++) { + Ref curr = arr[i]; + if (skip) { + arr[i-skip] = curr; + } + if (isEmpty(deStat(curr)) || (curr[0] == STAT && !hasSideEffects(curr[1]))) { + skip++; + } + } + if (skip) arr->setSize(arr->size() - skip); +} void removeAllEmptySubNodes(Ref ast) { traversePre(ast, [](Ref node) { @@ -842,6 +855,18 @@ void removeAllEmptySubNodes(Ref ast) { } }); } +void removeAllUselessSubNodes(Ref ast) { + traversePre(ast, [](Ref node) { + int index = -1; + if (node[0] == DEFUN) { + clearUselessNodes(node[3]); + } else if (node[0] == BLOCK && !!node[1]) { + clearUselessNodes(node[1]); + } else if (node[0] == SEQ && isEmpty(node[1])) { + safeCopy(node, node[2]); + } + }); +} Ref unVarify(Ref vars) { // transform var x=1, y=2 etc. into (x=1, y=2), i.e., the same assigns, but without a var definition Ref ret = makeArray(); @@ -2348,7 +2373,7 @@ void registerize(Ref ast) { allVars.insert(node[1]->getIString()); } }); - removeAllEmptySubNodes(fun); // vacuum? + removeAllUselessSubNodes(fun); // vacuum? StringTypeMap regTypes; // reg name -> type auto getNewRegName = [&](int num, IString name) { const char *str; From 582ce2a550f37896731855e2c19d24ecdbeaabf4 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Nov 2014 17:29:19 -0800 Subject: [PATCH 120/161] fix oob read on vector in registerize --- tools/optimizer/optimizer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index da60d02a44fca..671c35cef2b6c 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -2559,7 +2559,7 @@ void registerize(Ref ast) { Ref type = node[0]; if (LOOP.has(type)) { // Free registers that were locked to this loop - if (loopRegs[loops].size() > 0) { + if (loopRegs.size() > loops && loopRegs[loops].size() > 0) { for (auto loopReg : loopRegs[loops]) { freeRegsClasses[regTypes[loopReg]].push_back(loopReg); } From b2d3cadf151149f566981820d17f0ca452d33a71 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Nov 2014 17:53:31 -0800 Subject: [PATCH 121/161] optimizer fixes --- tools/optimizer/optimizer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 671c35cef2b6c..604e82c738524 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -237,6 +237,7 @@ void traversePrePostConditional(Ref node, std::function visitPre, st // Traverses all the top-level functions in the document void traverseFunctions(Ref ast, std::function visit) { + if (!ast || ast->size() == 0) return; if (ast[0] == TOPLEVEL) { Ref stats = ast[1]; for (int i = 0; i < stats->size(); i++) { @@ -860,7 +861,7 @@ void removeAllUselessSubNodes(Ref ast) { int index = -1; if (node[0] == DEFUN) { clearUselessNodes(node[3]); - } else if (node[0] == BLOCK && !!node[1]) { + } else if (node[0] == BLOCK && node->size() > 1 && !!node[1]) { clearUselessNodes(node[1]); } else if (node[0] == SEQ && isEmpty(node[1])) { safeCopy(node, node[2]); From 0a84b64a14d18e97860e3116ceaa05ee09293d25 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Nov 2014 17:56:54 -0800 Subject: [PATCH 122/161] save intermediate json stages with suffix json --- emcc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emcc b/emcc index 38dd2dea59d44..0db9be2a4f803 100755 --- a/emcc +++ b/emcc @@ -1389,7 +1389,7 @@ try: logging.debug('applying js optimization passes: %s', passes) final = shared.Building.js_optimizer(final, passes, jcache, debug_level >= 4, js_optimizer_extra_info, just_split=just_split, just_concat=just_concat) js_transform_tempfiles.append(final) - if DEBUG: save_intermediate(title) + if DEBUG: save_intermediate(title, suffix='js' if 'emitJSON' not in passes else 'json') passes = js_optimizer_queue[:] From f6e3a9c2304467901ec2625e1fdeb3d4c05e97d1 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Nov 2014 11:17:24 -0800 Subject: [PATCH 123/161] make array accesses non-tolerant --- tools/optimizer/minijson.h | 14 +++++++--- tools/optimizer/optimizer.cpp | 50 ++++++++++++++++++++--------------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index d853a9e279a20..2d06215f2b153 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -432,10 +432,9 @@ struct Value { } } - Ref& operator[](unsigned x) { // tolerant, returns Null on out of bounds access. makes it convenient to check e.g. [3] on an if node - static Ref null = arena.alloc(); // TODO: freeze this + Ref& operator[](unsigned x) { assert(isArray()); - if (x >= arr->size()) return null; + assert(x < arr->size()); return (*arr)[x]; } @@ -451,6 +450,12 @@ struct Value { return ret; } + Ref back() { + assert(isArray()); + if (arr->size() == 0) return nullptr; + return arr->back(); + } + void splice(int x, int num) { assert(isArray()); arr->erase(arr->begin() + x, arr->begin() + x + num); @@ -549,7 +554,8 @@ Ref Arena::alloc() { void dump(const char *str, Ref node, bool pretty) { std::cerr << str << ": "; - node->stringify(std::cerr, pretty); + if (!!node) node->stringify(std::cerr, pretty); + else std::cerr << "(nullptr)"; std::cerr << std::endl; } diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 604e82c738524..03bc7762f6775 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -172,6 +172,7 @@ struct StackedStack { // a stack, on the stack // Traverse, calling visit before the children void traversePre(Ref node, std::function visit) { + if (!visitable(node)) return; visit(node); StackedStack stack; stack.push_back({ node, 0 }); @@ -192,6 +193,7 @@ void traversePre(Ref node, std::function visit) { // Traverse, calling visitPre before the children and visitPost after void traversePrePost(Ref node, std::function visitPre, std::function visitPost) { + if (!visitable(node)) return; visitPre(node); StackedStack stack; stack.push_back({ node, 0 }); @@ -214,6 +216,7 @@ void traversePrePost(Ref node, std::function visitPre, std::function // Traverse, calling visitPre before the children and visitPost after. If pre returns false, do not traverse children void traversePrePostConditional(Ref node, std::function visitPre, std::function visitPost) { + if (!visitable(node)) return; if (!visitPre(node)) return; StackedStack stack; stack.push_back({ node, 0 }); @@ -258,7 +261,7 @@ Ref getStatements(Ref node) { if (node[0] == DEFUN) { return node[3]; } else if (node[0] == BLOCK) { - return node[1]; + return node->size() > 1 ? node[1] : nullptr; } else { return arena.alloc(); } @@ -373,7 +376,7 @@ struct AsmData { i++; } // look for final RETURN statement to get return type. - Ref retStmt = stats[stats->size() - 1]; + Ref retStmt = stats->back(); if (!!retStmt && retStmt[0] == RETURN && !!retStmt[1]) { ret = detectType(retStmt[1]); } else { @@ -431,7 +434,7 @@ struct AsmData { */ // ensure that there's a final RETURN statement if needed. if (ret != ASM_NONE) { - Ref retStmt = stats[stats->size() - 1]; + Ref retStmt = stats->back(); if (!retStmt || retStmt[0] != RETURN) { Ref retVal = makeNum(0); if (ret != ASM_INT) { @@ -849,7 +852,7 @@ void removeAllEmptySubNodes(Ref ast) { int index = -1; if (node[0] == DEFUN) { clearEmptyNodes(node[3]); - } else if (node[0] == BLOCK && !!node[1]) { + } else if (node[0] == BLOCK && node->size() > 1 && !!node[1]) { clearEmptyNodes(node[1]); } else if (node[0] == SEQ && isEmpty(node[1])) { safeCopy(node, node[2]); @@ -880,9 +883,12 @@ Ref unVarify(Ref vars) { // transform var x=1, y=2 etc. into (x=1, y=2), i.e., t for (int i = 0; i < vars->size()-1; i++) { curr->push_back(makeString(SEQ)); curr->push_back(make3(ASSIGN, &(arena.alloc())->setBool(true), makeName(vars[i][0]->getIString()), vars[i][1])); - if (i != vars->size()-2) curr = curr[2] = makeArray(); + if (i != vars->size()-2) { + curr->push_back(makeArray()); + curr = curr[2]; + } } - curr->push_back(make3(ASSIGN, &(arena.alloc())->setBool(true), makeName(vars[vars->size()-1][0]->getIString()), vars[vars->size()-1][1])); + curr->push_back(make3(ASSIGN, &(arena.alloc())->setBool(true), makeName(vars->back()[0]->getIString()), vars->back()[1])); } return ret; } @@ -1060,8 +1066,8 @@ void eliminate(Ref ast, bool memSafe=false) { for (int i = 0; i < node1->size(); i++) { Ref node1i = node1[i]; IString name = node1i[0]->getIString(); - Ref value = node1i[1]; - if (!!value) { + Ref value; + if (node1i->size() > 1 && !!(value = node1i[1])) { definitions[name]++; if (!values.has(name)) values[name] = value; } @@ -1309,8 +1315,8 @@ void eliminate(Ref ast, bool memSafe=false) { Ref vars = node[1]; for (int i = 0; i < vars->size(); i++) { IString name = vars[i][0]->getIString(); - Ref value = vars[i][1]; - if (!!value) { + Ref value; + if (vars[i]->size() > 1 && !!(value = vars[i][1])) { traverseInOrder(value, false, false); if (potentials.has(name) && allowTracking) { track(name, value, node); @@ -1386,7 +1392,7 @@ void eliminate(Ref ast, bool memSafe=false) { tracked.clear(); } } else if (type == BLOCK) { - Ref stats = node[1]; + Ref stats = getStatements(node); if (!!stats) { for (int i = 0; i < stats->size(); i++) { traverseInOrder(stats[i], false, false); @@ -1530,20 +1536,20 @@ void eliminate(Ref ast, bool memSafe=false) { } else if (type == WHILE) { // try to remove loop helper variables specifically Ref stats = node[2][1]; - Ref last = stats[stats->size()-1]; + Ref last = stats->back(); if (!!last && last[0] == IF && last[2][0] == BLOCK && !!last[3] && last[3][0] == BLOCK) { Ref ifTrue = last[2]; Ref ifFalse = last[3]; clearEmptyNodes(ifTrue[1]); clearEmptyNodes(ifFalse[1]); bool flip = false; - if (!!ifFalse[1][0] && ifFalse[1][ifFalse[1]->size()-1][0] == BREAK) { // canonicalize break in the if-true + if (ifFalse[1]->size() > 0 && !!ifFalse[1][0] && !!ifFalse[1]->back() && ifFalse[1]->back()[0] == BREAK) { // canonicalize break in the if-true Ref temp = ifFalse; ifFalse = ifTrue; ifTrue = temp; flip = true; } - if (!!ifTrue[1][0] && ifTrue[1][ifTrue[1]->size()-1][0] == BREAK) { + if (ifTrue[1]->size() > 0 && !!ifTrue[1][0] && !!ifTrue[1]->back() && ifTrue[1]->back()[0] == BREAK) { Ref assigns = ifFalse[1]; clearEmptyNodes(assigns); std::vector loopers, helpers; @@ -1798,8 +1804,8 @@ void simplifyExpressions(Ref ast) { // We might be able to remove this correction for (int i = stack.size()-1; i >= 0; i--) { if (stack[i] >= 1) { - if (stack[stack.size()-1] < 2 && node[2][0] == CALL) break; // we can only remove multiple |0s on these - if (stack[stack.size()-1] < 1 && (COERCION_REQUIRING_OPS.has(node[2][0]) || + if (stack.back() < 2 && node[2][0] == CALL) break; // we can only remove multiple |0s on these + if (stack.back() < 1 && (COERCION_REQUIRING_OPS.has(node[2][0]) || (node[2][0] == BINARY && COERCION_REQUIRING_BINARIES.has(node[2][1])))) break; // we can remove |0 or >>2 // we will replace ourselves with the non-zero side. Recursively process that node. Ref result = node[2][0] == NUM && node[2][1]->getNumber() == 0 ? node[3] : node[2], other; @@ -2181,13 +2187,13 @@ void simplifyIfs(Ref ast) { while (body[0] == BLOCK) { Ref stats = body[1]; if (stats->size() == 0) break; - Ref other = stats[stats->size()-1]; + Ref other = stats->back(); if (other[0] != IF) { // our if block does not end with an if. perhaps if have an else we can flip - if (!!node[3] && node[3][0] == BLOCK) { + if (node->size() > 3 && !!node[3] && node[3][0] == BLOCK) { stats = node[3][1]; if (stats->size() == 0) break; - other = stats[stats->size()-1]; + other = stats->back(); if (other[0] == IF) { // flip node node[1] = flipCondition(node[1]); @@ -2279,11 +2285,11 @@ void simplifyIfs(Ref ast) { traversePrePost(func, [&inLoop, &labelAssigns, &labelChecks](Ref node) { if (node[0] == WHILE) inLoop++; Ref stats = getStatements(node); - if (!stats->isNull() && stats->size() > 0) { + if (!!stats && stats->size() > 0) { for (int i = 0; i < stats->size()-1; i++) { Ref pre = stats[i]; Ref post = stats[i+1]; - if (pre[0] == IF && !!pre[3] && post[0] == IF && !post[3]) { + if (pre[0] == IF && pre->size() > 3 && !!pre[3] && post[0] == IF && (post->size() <= 3 || !post[3])) { Ref postCond = post[1]; if (postCond[0] == BINARY && postCond[1] == EQ && postCond[2][0] == BINARY && postCond[2][1] == OR && @@ -2455,7 +2461,7 @@ void registerize(Ref ast) { level++; traversePrePostConditional(node[2], possibilifier, [](Ref node){}); purgeLevel(); - if (!!node[3]) { + if (node->size() > 3 && !!node[3]) { level++; traversePrePostConditional(node[3], possibilifier, [](Ref node){}); purgeLevel(); From 68873d79a2b43d2f424f541b7afe09d18ea4b066 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Nov 2014 13:10:27 -0800 Subject: [PATCH 124/161] fix chunkification on large json input --- tools/js_optimizer.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 8252a8387f484..04a767334c16b 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -245,12 +245,17 @@ def split_funcs(js, just_split=False): # if we are making source maps, we want our debug numbering to start from the # top of the file, so avoid breaking the JS into chunks cores = 1 if source_map else int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) - intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE)) - chunk_size = min(MAX_CHUNK_SIZE, max(MIN_CHUNK_SIZE, total_size / intended_num_chunks)) - chunks = shared.chunkify(funcs, chunk_size, jcache.get_cachename('jsopt') if jcache else None) + if not just_split: + intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE)) + chunk_size = min(MAX_CHUNK_SIZE, max(MIN_CHUNK_SIZE, total_size / intended_num_chunks)) + chunks = shared.chunkify(funcs, chunk_size, jcache.get_cachename('jsopt') if jcache else None) + else: + # keep same chunks as before + chunks = map(lambda f: f[1], funcs) + chunks = filter(lambda chunk: len(chunk) > 0, chunks) - if DEBUG and len(chunks) > 0: print >> sys.stderr, 'chunkification: intended size:', chunk_size, 'num funcs:', len(funcs), 'actual num chunks:', len(chunks), 'chunk size range:', max(map(len, chunks)), '-', min(map(len, chunks)) + if DEBUG and len(chunks) > 0: print >> sys.stderr, 'chunkification: num funcs:', len(funcs), 'actual num chunks:', len(chunks), 'chunk size range:', max(map(len, chunks)), '-', min(map(len, chunks)) funcs = None if jcache: @@ -306,12 +311,12 @@ def write_chunk(chunk, i): cores = min(cores, len(filenames)) if len(chunks) > 1 and cores >= 2: # We can parallelize - if DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks of size %d, using %d cores (total: %.2f MB)' % (len(chunks), chunk_size, cores, total_size/(1024*1024.)) + if DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks, using %d cores (total: %.2f MB)' % (len(chunks), cores, total_size/(1024*1024.)) pool = multiprocessing.Pool(processes=cores) filenames = pool.map(run_on_chunk, commands, chunksize=1) else: # We can't parallize, but still break into chunks to avoid uglify/node memory issues - if len(chunks) > 1 and DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks of size %d' % (len(chunks), chunk_size) + if len(chunks) > 1 and DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks' % (len(chunks)) filenames = [run_on_chunk(command) for command in commands] else: filenames = [] From 2d53599913b7b3c66deeb7b10ad4b6f62bf705cf Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Nov 2014 13:47:27 -0800 Subject: [PATCH 125/161] support for JSON objects and extraInfo in optimizer --- tools/optimizer/minijson.h | 95 ++++++++++++++++++++++++++++++++--- tools/optimizer/optimizer.cpp | 9 +++- 2 files changed, 97 insertions(+), 7 deletions(-) diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index 2d06215f2b153..ef3a679ae7fbf 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -157,18 +157,21 @@ struct Value { Number = 1, Array = 2, Null = 3, - Bool = 4 + Bool = 4, + Object = 5 }; Type type; typedef std::vector ArrayStorage; + typedef std::unordered_map ObjectStorage; union { // TODO: optimize IString str; double num; ArrayStorage *arr; bool boo; + ObjectStorage *obj; }; // constructors all copy their input @@ -191,6 +194,7 @@ struct Value { void free() { if (type == Array) delete arr; + else if (type == Object) delete obj; type = Null; num = 0; } @@ -237,12 +241,19 @@ struct Value { boo = b; return *this; } + Value& setObject() { + free(); + type = Object; + obj = new ObjectStorage(); + return *this; + } bool isString() { return type == String; } bool isNumber() { return type == Number; } bool isArray() { return type == Array; } bool isNull() { return type == Null; } bool isBool() { return type == Bool; } + bool isObject() { return type == Object; } bool isBool(bool b) { return type == Bool && b == boo; } // avoid overloading == as it might overload over int @@ -285,6 +296,8 @@ struct Value { case Bool: setBool(other.boo); break; + case Object: + assert(0); // TODO } return *this; } @@ -302,6 +315,8 @@ struct Value { break; case Bool: return boo == other.boo; + case Object: + return this == &other; // if you want a deep compare, use deepCompare } return true; } @@ -310,12 +325,21 @@ struct Value { Value& other = *ref; if (*this == other) return true; // either same pointer, or identical value type (string, number, null or bool) if (type != other.type) return false; - if (type != Array) return false; // Array is the only one where deep compare differs makes sense, others are shallow and were already tested - if (arr->size() != other.arr->size()) return false; - for (unsigned i = 0; i < arr->size(); i++) { - if (!(*arr)[i]->deepCompare((*other.arr)[i])) return false; + if (type == Array) { + if (arr->size() != other.arr->size()) return false; + for (unsigned i = 0; i < arr->size(); i++) { + if (!(*arr)[i]->deepCompare((*other.arr)[i])) return false; + } + return true; + } else if (type == Object) { + if (obj->size() != other.obj->size()) return false; + for (auto i : *obj) { + if (other.obj->count(i.first) == 0) return false; + if (i.second->deepCompare((*other.obj)[i.first])) return false; + } + return true; } - return true; + return false; } char* parse(char* curr) { @@ -361,6 +385,33 @@ struct Value { assert(strncmp(curr, "false", 5) == 0); setBool(false); curr += 5; + } else if (*curr == '{') { + // Object + curr++; + skip(); + setObject(); + while (*curr != '}') { + assert(*curr == '"'); + curr++; + char *close = strchr(curr, '"'); + assert(close); + *close = 0; // end this string, and reuse it straight from the input + IString key(curr); + curr = close+1; + skip(); + assert(*curr == ':'); + curr++; + skip(); + Ref value = arena.alloc(); + curr = value->parse(curr); + (*obj)[key] = value; + skip(); + if (*curr == '}') break; + assert(*curr == ','); + curr++; + skip(); + } + curr++; } else { // Number char *after; @@ -407,6 +458,31 @@ struct Value { case Bool: os << (boo ? "true" : "false"); break; + case Object: + os << '{'; + if (pretty) { + os << std::endl; + indent++; + } + bool first = true; + for (auto i : *obj) { + if (first) { + first = false; + } else { + os << ", "; + if (pretty) os << std::endl; + } + indentify(); + os << '"' << i.first.c_str() << "\": "; + i.second->stringify(os, pretty); + } + if (pretty) { + os << std::endl; + indent--; + } + indentify(); + os << '}'; + break; } } @@ -508,6 +584,13 @@ struct Value { // Null operations // Bool operations + + // Object operations + + Ref& operator[](IString x) { + assert(isObject()); + return (*obj)[x]; + } }; // Ref methods diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 03bc7762f6775..cf75f3d8eb533 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -10,7 +10,7 @@ // Globals //================== -Ref doc; +Ref doc, extraInfo; IString TOPLEVEL("toplevel"), DEFUN("defun"), @@ -2619,7 +2619,14 @@ int main(int argc, char **argv) { json[size] = 0; char *comment = strstr(json, "//"); + char *extraInfoStart = strstr(json, "// EXTRA_INFO:"); if (comment) *comment = 0; // drop off the comments; TODO: parse extra info + if (extraInfoStart) { + extraInfo = arena.alloc(); + extraInfo->parse(extraInfoStart + 14); + extraInfo->stringify(std::cout, true); + std::cout << "............\n"; + } // Parse JSON source into the document doc = arena.alloc(); From 92fbdce5212d2ba9bb6c2c3bbbf21a030755384a Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Nov 2014 14:39:56 -0800 Subject: [PATCH 126/161] initial port of minifyLocals --- tests/test_other.py | 5 ++ tools/optimizer/minijson.h | 10 +++ tools/optimizer/optimizer.cpp | 151 +++++++++++++++++++++++++++++++++- 3 files changed, 162 insertions(+), 4 deletions(-) diff --git a/tests/test_other.py b/tests/test_other.py index 94d884aff7a5c..6f7423ee6b844 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -1946,6 +1946,11 @@ def test_js_optimizer(self): output_temp = 'output.js' shutil.copyfile(input, input_temp) Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), input_temp, 'emitJSON'], stdin=PIPE, stdout=open(input_temp + '.js', 'w')).communicate() + original = open(input).read() + if '// EXTRA_INFO:' in original: + json = open(input_temp + '.js').read() + json += '\n' + original[original.find('// EXTRA_INFO:'):] + open(input_temp + '.js', 'w').write(json) output = Popen([js_optimizer.get_native_optimizer(), input_temp + '.js'] + passes, stdin=PIPE, stdout=open(output_temp, 'w')).communicate()[0] Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), output_temp, 'receiveJSON'], stdin=PIPE, stdout=open(output_temp + '.js', 'w')).communicate() output = open(output_temp + '.js').read() diff --git a/tools/optimizer/minijson.h b/tools/optimizer/minijson.h index ef3a679ae7fbf..3745e1549adb0 100644 --- a/tools/optimizer/minijson.h +++ b/tools/optimizer/minijson.h @@ -40,6 +40,7 @@ struct Ref { Value& operator*() { return *inst; } Value* operator->() { return inst; } Ref& operator[](unsigned x); + Ref& operator[](IString x); // special conveniences bool operator==(const char *str); // comparison to string, which is by value @@ -591,6 +592,11 @@ struct Value { assert(isObject()); return (*obj)[x]; } + + bool has(IString x) { + assert(isObject()); + return obj->count(x) > 0; + } }; // Ref methods @@ -599,6 +605,10 @@ Ref& Ref::operator[](unsigned x) { return (*get())[x]; } +Ref& Ref::operator[](IString x) { + return (*get())[x]; +} + bool Ref::operator==(const char *str) { return get()->isString() && !strcmp(get()->str.str, str); } diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index cf75f3d8eb533..191472a9dfaf7 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -6,6 +6,8 @@ #include "minijson.h" +typedef std::vector StringVec; + //================== // Globals //================== @@ -2401,7 +2403,7 @@ void registerize(Ref ast) { temp[written] = 0; IString ret(temp); // likely interns a new string; leaks if not XXX FIXME regTypes[ret] = type; - assert(!allVars.has(ret)); // register must not shadow non-local name + assert(!allVars.has(ret) || asmData.isLocal(ret)); // register must not shadow non-local name return ret; }; // Find the # of uses of each variable. @@ -2496,7 +2498,6 @@ void registerize(Ref ast) { // we just use a fresh register to make sure we avoid this, but it could be // optimized to check for safe registers (free, and not used in this loop level). StringStringMap varRegs; // maps variables to the register they will use all their life - typedef std::vector StringVec; std::vector freeRegsClasses; freeRegsClasses.resize(ASM_NONE); int nextReg = 1; @@ -2597,6 +2598,149 @@ void registerize(Ref ast) { }); } +// minified names generation +StringSet RESERVED("do if in for new try var env let"); +const char *VALID_MIN_INITS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_$"; +const char *VALID_MIN_LATERS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_$0123456789"; + +StringVec minifiedNames; +std::vector minifiedState; + +void ensureMinifiedNames(int n) { // make sure the nth index in minifiedNames exists. done 100% deterministically + static int VALID_MIN_INITS_LEN = strlen(VALID_MIN_INITS); + static int VALID_MIN_LATERS_LEN = strlen(VALID_MIN_LATERS); + + while (minifiedNames.size() < n+1) { + // generate the current name + std::string name; + name += VALID_MIN_INITS[minifiedState[0]]; + for (int i = 1; i < minifiedState.size(); i++) { + name += VALID_MIN_LATERS[minifiedState[i]]; + } + IString str(name.c_str()); + if (!RESERVED.has(str)) minifiedNames.push_back(str); + // increment the state + int i = 0; + while (1) { + minifiedState[i]++; + if (minifiedState[i] < (i == 0 ? VALID_MIN_INITS_LEN : VALID_MIN_LATERS_LEN)) break; + // overflow + minifiedState[i] = 0; + i++; + if (i == minifiedState.size()) minifiedState.push_back(-1); // will become 0 after increment in next loop head + } + } +} + +void minifyLocals(Ref ast) { + assert(!!extraInfo); + IString GLOBALS("globals"); + assert(extraInfo->has(GLOBALS)); + Ref globals = extraInfo[GLOBALS]; + + if (minifiedState.size() == 0) minifiedState.push_back(0); + + traverseFunctions(ast, [&globals](Ref fun) { + // Analyse the asmjs to figure out local variable names, + // but operate on the original source tree so that we don't + // miss any global names in e.g. variable initializers. + AsmData asmData(fun); + asmData.denormalize(); // TODO: we can avoid modifying at all here - we just need a list of local vars+params + + StringStringMap newNames; + StringSet usedNames; + + // Find all the globals that we need to minify using + // pre-assigned names. Don't actually minify them yet + // as that might interfere with local variable names. + traversePre(fun, [&](Ref node) { + if (node[0] == NAME) { + IString name = node[1]->getIString(); + if (!asmData.isLocal(name)) { + IString minified = globals[name]->getIString(); + if (!!minified) { + newNames[name] = minified; + usedNames.insert(minified); + } + } + } + }); + + // The first time we encounter a local name, we assign it a + // minified name that's not currently in use. Allocating on + // demand means they're processed in a predictable order, + // which is very handy for testing/debugging purposes. + int nextMinifiedName = 0; + auto getNextMinifiedName = [&]() { + IString minified; + while (1) { + ensureMinifiedNames(nextMinifiedName); + minified = minifiedNames[nextMinifiedName++]; + // TODO: we can probably remove !isLocalName here + if (!usedNames.has(minified) && !asmData.isLocal(minified)) { + return minified; + } + } + }; + + // We can also minify loop labels, using a separate namespace + // to the variable declarations. + StringStringMap newLabels; + int nextMinifiedLabel = 0; + auto getNextMinifiedLabel = [&]() { + ensureMinifiedNames(nextMinifiedLabel); + return minifiedNames[nextMinifiedLabel++]; + }; + + // Traverse and minify all names. + if (globals->has(fun[1]->getIString())) { + fun[1]->setString(globals[fun[1]->getIString()]->getIString()); + assert(!!fun[1]); + } + if (!!fun[2]) { + for (int i = 0; i < fun[2]->size(); i++) { + IString minified = getNextMinifiedName(); + newNames[fun[2][i]->getIString()] = minified; + fun[2][i]->setString(minified); + } + } + traversePre(fun[3], [&](Ref node) { + Ref type = node[0]; + if (type == NAME) { + IString name = node[1]->getIString(); + IString minified = newNames[name]; + if (!!minified) { + node[1]->setString(minified); + } else if (asmData.isLocal(name)) { + minified = getNextMinifiedName(); + newNames[name] = minified; + node[1]->setString(minified); + } + } else if (type == VAR) { + for (int i = 0; i < node[1]->size(); i++) { + Ref defn = node[1][i]; + IString name = defn[0]->getIString(); + if (!(newNames.has(name))) { + newNames[name] = getNextMinifiedName(); + } + defn[0]->setString(newNames[name]); + } + } else if (type == LABEL) { + IString name = node[1]->getIString(); + if (!newLabels[name]) { + newLabels[name] = getNextMinifiedLabel(); + } + node[1]->setString(newLabels[name]); + } else if ((type == BREAK || type == CONTINUE) && node->size() > 1) { + IString name = node[1]->getIString(); + if (!!name) { + name = newLabels[name]; + } + } + }); + }); +} + //================== // Main //================== @@ -2624,8 +2768,6 @@ int main(int argc, char **argv) { if (extraInfoStart) { extraInfo = arena.alloc(); extraInfo->parse(extraInfoStart + 14); - extraInfo->stringify(std::cout, true); - std::cout << "............\n"; } // Parse JSON source into the document @@ -2645,6 +2787,7 @@ int main(int argc, char **argv) { else if (str == "optimizeFrounds") optimizeFrounds(doc); else if (str == "simplifyIfs") simplifyIfs(doc); else if (str == "registerize") registerize(doc); + else if (str == "minifyLocals") minifyLocals(doc); else { fprintf(stderr, "unrecognized argument: %s\n", str.c_str()); assert(0); From e6ff252953dc021c04cabd5f2ccdfa52a3b4e1e4 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Nov 2014 15:01:19 -0800 Subject: [PATCH 127/161] minifyLocals fixes --- tools/optimizer/optimizer.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 191472a9dfaf7..1d04145debbf4 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -2617,7 +2617,7 @@ void ensureMinifiedNames(int n) { // make sure the nth index in minifiedNames ex for (int i = 1; i < minifiedState.size(); i++) { name += VALID_MIN_LATERS[minifiedState[i]]; } - IString str(name.c_str()); + IString str(strdupe(name.c_str())); // leaked! if (!RESERVED.has(str)) minifiedNames.push_back(str); // increment the state int i = 0; @@ -2727,14 +2727,13 @@ void minifyLocals(Ref ast) { } } else if (type == LABEL) { IString name = node[1]->getIString(); - if (!newLabels[name]) { + if (!newLabels.has(name)) { newLabels[name] = getNextMinifiedLabel(); } node[1]->setString(newLabels[name]); - } else if ((type == BREAK || type == CONTINUE) && node->size() > 1) { - IString name = node[1]->getIString(); - if (!!name) { - name = newLabels[name]; + } else if (type == BREAK || type == CONTINUE) { + if (node->size() > 1 && !!node[1]) { + node[1]->setString(newLabels[node[1]->getIString()]); } } }); From f8750120dc544ab47e60068be6ff130033f72358 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Nov 2014 15:38:39 -0800 Subject: [PATCH 128/161] fix and enable minifyNames in native optimizer --- tools/js_optimizer.py | 8 +++++--- tools/optimizer/optimizer.cpp | 5 +++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 04a767334c16b..7b8ccdfff26af 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -9,7 +9,7 @@ def path_from_root(*pathelems): return os.path.join(__rootpath__, *pathelems) -NATIVE_PASSES = set(['asm', 'asmPreciseF32', 'receiveJSON', 'emitJSON', 'eliminate', 'eliminateMemSafe', 'simplifyExpressions', 'simplifyIfs', 'optimizeFrounds', 'registerize']) +NATIVE_PASSES = set(['asm', 'asmPreciseF32', 'receiveJSON', 'emitJSON', 'eliminate', 'eliminateMemSafe', 'simplifyExpressions', 'simplifyIfs', 'optimizeFrounds', 'registerize', 'minifyNames', 'minifyLocals']) JS_OPTIMIZER = path_from_root('tools', 'js-optimizer.js') @@ -106,8 +106,10 @@ def run_on_chunk(command): filename = command[index + 1] else: filename = command[1] - #print >> sys.stderr, 'running js optimizer command', ' '.join(map(lambda c: c if c != filename else 'input.txt', command)) - #shutil.copyfile(filename, '/tmp/emscripten_temp/input.txt') + #saved = 'save_' + os.path.basename(filename) + #while os.path.exists(saved): saved = 'input' + str(int(saved.replace('input', '').replace('.txt', ''))+1) + '.txt' + #print >> sys.stderr, 'running js optimizer command', ' '.join(map(lambda c: c if c != filename else saved, command)) + #shutil.copyfile(filename, os.path.join('/tmp/emscripten_temp', saved)) output = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0] assert len(output) > 0 and not output.startswith('Assertion failed'), 'Error in js optimizer: ' + output filename = temp_files.get(os.path.basename(filename) + '.jo.js').name diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 1d04145debbf4..6f4c8e0fa2887 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -2657,8 +2657,9 @@ void minifyLocals(Ref ast) { if (node[0] == NAME) { IString name = node[1]->getIString(); if (!asmData.isLocal(name)) { - IString minified = globals[name]->getIString(); - if (!!minified) { + if (globals->has(name)) { + IString minified = globals[name]->getIString(); + assert(!!minified); newNames[name] = minified; usedNames.insert(minified); } From e900594be795763d0b065c3012691823b4816640 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Nov 2014 15:56:04 -0800 Subject: [PATCH 129/161] refactor decision to run in native optimizer --- emcc | 2 +- tests/test_other.py | 2 +- tools/js_optimizer.py | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/emcc b/emcc index 0db9be2a4f803..e3c570e25788f 100755 --- a/emcc +++ b/emcc @@ -1400,7 +1400,7 @@ try: curr = [] native = False for p in passes: - if not shared.js_optimizer.use_native([p], source_map=debug_level >= 4): + if not shared.js_optimizer.use_native(p, source_map=debug_level >= 4): if native: chunks.append(['receiveJSON'] + curr + ['emitJSON']) curr = [] diff --git a/tests/test_other.py b/tests/test_other.py index 6f7423ee6b844..c8c0d64dad46b 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -1938,7 +1938,7 @@ def test_js_optimizer(self): print ' js' output = Popen(listify(NODE_JS) + [path_from_root('tools', 'js-optimizer.js'), input] + passes, stdin=PIPE, stdout=PIPE).communicate()[0] self.assertIdentical(expected, output.replace('\r\n', '\n').replace('\n\n', '\n')) - if js_optimizer.use_native(passes) and 'asm' in passes: + if js_optimizer.use_native(passes): # test calling native print ' native' self.clear() diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 7b8ccdfff26af..8439c1259014d 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -34,8 +34,11 @@ def create_optimizer(): return output return shared.Cache.get('optimizer.exe', create_optimizer, extension='exe') -def use_native(passes, source_map=False): - return (not source_map) and len(NATIVE_PASSES.intersection(passes)) == len(passes) +# Check if we should run a pass or set of passes natively. if a set of passes, they must all be valid to run in the native optimizer at once. +def use_native(x, source_map=False): + if source_map: return False; + if type(x) == str: return x in NATIVE_PASSES + return len(NATIVE_PASSES.intersection(x)) == len(x) and 'asm' in x class Minifier: ''' From 3e0030c61584eed2fcddf5724733d0d428ec60ea Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Nov 2014 15:58:16 -0800 Subject: [PATCH 130/161] put native optimizer behind env var --- tools/js_optimizer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 8439c1259014d..c91b6a0bb2598 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -24,6 +24,8 @@ def path_from_root(*pathelems): func_sig = re.compile('function ([_\w$]+)\(') import_sig = re.compile('var ([_\w$]+) *=[^;]+;') +NATIVE_OPTIMIZER = os.environ.get('EMCC_NATIVE_OPTIMIZER') + def get_native_optimizer(): def create_optimizer(): shared.logging.debug('building native optimizer') @@ -36,7 +38,8 @@ def create_optimizer(): # Check if we should run a pass or set of passes natively. if a set of passes, they must all be valid to run in the native optimizer at once. def use_native(x, source_map=False): - if source_map: return False; + if source_map: return False + if not NATIVE_OPTIMIZER or NATIVE_OPTIMIZER == '0': return False if type(x) == str: return x in NATIVE_PASSES return len(NATIVE_PASSES.intersection(x)) == len(x) and 'asm' in x From 31b429f38a16bf2a9b40dc92e9258d503c5d68ad Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Nov 2014 16:04:38 -0800 Subject: [PATCH 131/161] log out when using native optimizer --- tools/js_optimizer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index c91b6a0bb2598..613d4002005e9 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -312,6 +312,7 @@ def write_chunk(chunk, i): (['--debug'] if source_map else []) + passes, filenames) else: # use the native optimizer + shared.logging.debug('js optimizer using native') assert not source_map # XXX need to use js optimizer commands = map(lambda filename: [get_native_optimizer(), filename] + passes, filenames) #print [' '.join(command) for command in commands] From 772d896fcf8349696b01683f8a00bbd87b7a4678 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Nov 2014 17:45:34 -0800 Subject: [PATCH 132/161] enable and fix some minor tests that got disabled by mistake a long time ago --- tests/test_other.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_other.py b/tests/test_other.py index d4767e4c77309..c356e04892c65 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -192,11 +192,10 @@ def test_emcc(self): (['-O2'], lambda generated: '// The Module object' not in generated, 'with opts, no comments in shell code'), (['-O2', '-g2'], lambda generated: '// The Module object' not in generated, 'with -g2, no comments in shell code'), (['-O2', '-g3'], lambda generated: '// The Module object' in generated, 'with -g3, yes comments in shell code'), - (['-O2', '--profiling'], lambda generated: '// The Module object' in generated or os.environ.get('EMCC_FAST_COMPILER') == '0', 'with --profiling, yes comments in shell code (in fastcomp)'), ]: print params, text self.clear() - if os.environ.get('EMCC_FAST_COMPILER') != '0' and ['disable typed arrays', 'typed arrays 1 selected']: continue + if os.environ.get('EMCC_FAST_COMPILER') != '0' and text in ['disable typed arrays', 'typed arrays 1 selected']: continue output = Popen([PYTHON, compiler, path_from_root('tests', 'hello_world_loop.cpp'), '-o', 'a.out.js'] + params, stdout=PIPE, stderr=PIPE).communicate() assert len(output[0]) == 0, output[0] assert os.path.exists('a.out.js'), '\n'.join(output) From 3bdcddf316fea4e0d9ad43cd2659a1457da46880 Mon Sep 17 00:00:00 2001 From: Aleksander Guryanov Date: Fri, 14 Nov 2014 23:25:09 +0700 Subject: [PATCH 133/161] SDL_ttf add TTF_SizeUTF8 alias --- src/library_sdl.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/library_sdl.js b/src/library_sdl.js index 84f7d168edd63..a008bd68ec1d6 100644 --- a/src/library_sdl.js +++ b/src/library_sdl.js @@ -2914,6 +2914,7 @@ var LibrarySDL = { TTF_RenderText_Blended: 'TTF_RenderText_Solid', // XXX ignore blending vs. solid TTF_RenderText_Shaded: 'TTF_RenderText_Solid', // XXX ignore blending vs. solid TTF_RenderUTF8_Solid: 'TTF_RenderText_Solid', + TTF_SizeUTF8: 'TTF_SizeText', TTF_SizeText: function(font, text, w, h) { var fontData = SDL.fonts[font]; From 84f33288bff059db5aed80977651102309819e72 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Nov 2014 12:55:54 -0800 Subject: [PATCH 134/161] make test_source_map ignore the generated functions marker --- tests/test_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_core.py b/tests/test_core.py index 421f11d6adb4a..e2982e416eb54 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -6733,6 +6733,7 @@ def build_and_check(): # optimizer can deal with both types. out_file = re.sub(' *//[@#].*$', '', out_file, flags=re.MULTILINE) def clean(code): + code = code.replace('// EMSCRIPTEN_GENERATED_FUNCTIONS: ["_malloc","__Z3foov","_free","_main"]', '') code = re.sub(';', ';\n', code) # put statements each on a new line code = re.sub(r'\n+[ \n]*\n+', '\n', code) code = re.sub(' L\d+ ?:', '', code) # ignore labels; they can change in each compile From 2a7ffe1275991ba3ab7f92ebcc477218a37f485d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Nov 2014 13:10:02 -0800 Subject: [PATCH 135/161] detect function names in json input in minifier --- tools/js_optimizer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 613d4002005e9..ce93cb349866a 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -22,6 +22,7 @@ def path_from_root(*pathelems): DEBUG = os.environ.get('EMCC_DEBUG') func_sig = re.compile('function ([_\w$]+)\(') +func_sig_json = re.compile('\["defun", ?"([_\w$]+)",') import_sig = re.compile('var ([_\w$]+) *=[^;]+;') NATIVE_OPTIMIZER = os.environ.get('EMCC_NATIVE_OPTIMIZER') @@ -66,6 +67,8 @@ def minify_shell(self, shell, minify_whitespace, source_map=False): # Find all globals in the JS functions code self.globs = [m.group(1) for m in func_sig.finditer(self.js)] + if len(self.globs) == 0: + self.globs = [m.group(1) for m in func_sig_json.finditer(self.js)] temp_file = temp_files.get('.minifyglobals.js').name f = open(temp_file, 'w') From daf7715b58a350d6c8b0f2b6b9723480b30317be Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Nov 2014 13:21:58 -0800 Subject: [PATCH 136/161] turn minifyWhitespace into a global option in emcc, so that it is applied on all passes being run --- emcc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/emcc b/emcc index 861a9afa57de7..3920844996f76 100755 --- a/emcc +++ b/emcc @@ -1369,6 +1369,7 @@ try: js_optimizer_extra_info = {} js_optimizer_queue_history = [] js_optimizer_blacklist = (os.environ.get('EMCC_JSOPT_BLACKLIST') or '').split(',') + minify_whitespace = False def flush_js_optimizer_queue(title='js_opts'): global final, js_optimizer_queue, js_optimizer_extra_info, js_optimizer_queue_history @@ -1381,11 +1382,13 @@ try: if len(js_optimizer_queue) > 0 and not(not shared.Settings.ASM_JS and len(js_optimizer_queue) == 1 and js_optimizer_queue[0] == 'last'): def run_passes(passes, title, just_split, just_concat): - global final + global final, minify_whitespace if shared.Settings.ASM_JS: passes = ['asm'] + passes if shared.Settings.PRECISE_F32: passes = ['asmPreciseF32'] + passes + if minify_whitespace: + passes += ['minifyWhitespace'] logging.debug('applying js optimization passes: %s', passes) final = shared.Building.js_optimizer(final, passes, jcache, debug_level >= 4, js_optimizer_extra_info, just_split=just_split, just_concat=just_concat) js_transform_tempfiles.append(final) @@ -1483,7 +1486,9 @@ try: if debug_level < 2 and shared.Settings.ASM_JS and not closure == 2: js_optimizer_queue += ['minifyNames'] if emit_symbol_map: js_optimizer_queue += ['symbolMap='+target+'.symbols'] - if debug_level == 0: js_optimizer_queue += ['minifyWhitespace'] + if debug_level == 0: + global minify_whitespace + minify_whitespace = True if shared.Settings.ASM_JS: if closure == 1: From 7cd1439cfaa2aba7b8b65ec38ad2655f8e81b53d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Nov 2014 13:27:16 -0800 Subject: [PATCH 137/161] handle symbol map as an option on top of minifyNames --- emcc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/emcc b/emcc index 3920844996f76..17c8753a9b2f5 100755 --- a/emcc +++ b/emcc @@ -1382,11 +1382,13 @@ try: if len(js_optimizer_queue) > 0 and not(not shared.Settings.ASM_JS and len(js_optimizer_queue) == 1 and js_optimizer_queue[0] == 'last'): def run_passes(passes, title, just_split, just_concat): - global final, minify_whitespace + global final, minify_whitespace, emit_symbol_map, target if shared.Settings.ASM_JS: passes = ['asm'] + passes if shared.Settings.PRECISE_F32: passes = ['asmPreciseF32'] + passes + if emit_symbol_map and 'minifyNames' in passes: + passes += ['symbolMap='+target+'.symbols'] if minify_whitespace: passes += ['minifyWhitespace'] logging.debug('applying js optimization passes: %s', passes) @@ -1485,7 +1487,6 @@ try: if opt_level >= 2: if debug_level < 2 and shared.Settings.ASM_JS and not closure == 2: js_optimizer_queue += ['minifyNames'] - if emit_symbol_map: js_optimizer_queue += ['symbolMap='+target+'.symbols'] if debug_level == 0: global minify_whitespace minify_whitespace = True From 37aac8c430796c9416e617a36e43b9284c63927b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Nov 2014 16:15:52 -0800 Subject: [PATCH 138/161] clean up some asm.js code in emscripten.py --- emscripten.py | 950 +++++++++++++++++++++++++------------------------- 1 file changed, 469 insertions(+), 481 deletions(-) diff --git a/emscripten.py b/emscripten.py index 6ae75282c5b77..594da72b41b1b 100755 --- a/emscripten.py +++ b/emscripten.py @@ -757,7 +757,7 @@ def emscript_fast(infile, settings, outfile, libraries=[], compiler_engine=None, outfile: The file where the output is written. """ - assert(settings['ASM_JS']) + assert settings['ASM_JS'], 'fastcomp is asm.js-only (mode 1 or 2)' success = False @@ -925,14 +925,12 @@ def save_settings(): %s''' % (staticbump, global_initializers, mem_init)) # XXX wrong size calculation! funcs_js = [funcs] - if settings.get('ASM_JS'): - parts = pre.split('// ASM_LIBRARY FUNCTIONS\n') - if len(parts) > 1: - pre = parts[0] - funcs_js.append(parts[1]) + parts = pre.split('// ASM_LIBRARY FUNCTIONS\n') + if len(parts) > 1: + pre = parts[0] + funcs_js.append(parts[1]) # merge forwarded data - assert settings.get('ASM_JS'), 'fastcomp is asm.js only' settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS'] all_exported_functions = set(settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise for additional_export in settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented @@ -963,510 +961,500 @@ def save_settings(): #if DEBUG: outfile.write('// funcs\n') - if settings.get('ASM_JS'): - # when emulating function pointer casts, we need to know what is the target of each pointer - if settings['EMULATE_FUNCTION_POINTER_CASTS']: - function_pointer_targets = {} - for sig, table in last_forwarded_json['Functions']['tables'].iteritems(): - start = table.index('[') - end = table.rindex(']') - body = table[start+1:end].split(',') - parsed = map(lambda x: x.strip(), body) - for i in range(len(parsed)): - if parsed[i] != '0': - assert i not in function_pointer_targets - function_pointer_targets[i] = [sig, str(parsed[i])] - - # Move preAsms to their right place - def move_preasm(m): - contents = m.groups(0)[0] - outfile.write(contents + '\n') - return '' - funcs_js[1] = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m), funcs_js[1]) - - funcs_js += ['\n// EMSCRIPTEN_END_FUNCS\n'] - - class Counter: - i = 0 - j = 0 - if 'pre' in last_forwarded_json['Functions']['tables']: - pre_tables = last_forwarded_json['Functions']['tables']['pre'] - del last_forwarded_json['Functions']['tables']['pre'] - else: - pre_tables = '' + # when emulating function pointer casts, we need to know what is the target of each pointer + if settings['EMULATE_FUNCTION_POINTER_CASTS']: + function_pointer_targets = {} + for sig, table in last_forwarded_json['Functions']['tables'].iteritems(): + start = table.index('[') + end = table.rindex(']') + body = table[start+1:end].split(',') + parsed = map(lambda x: x.strip(), body) + for i in range(len(parsed)): + if parsed[i] != '0': + assert i not in function_pointer_targets + function_pointer_targets[i] = [sig, str(parsed[i])] - def unfloat(s): - return 'd' if s == 'f' else s # lower float to double for ffis + # Move preAsms to their right place + def move_preasm(m): + contents = m.groups(0)[0] + outfile.write(contents + '\n') + return '' + funcs_js[1] = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m), funcs_js[1]) - if settings['ASSERTIONS'] >= 2: - debug_tables = {} - - def make_params(sig): return ','.join(['p%d' % p for p in range(len(sig)-1)]) - def make_coerced_params(sig): return ','.join([shared.JS.make_coercion('p%d', unfloat(sig[p+1]), settings) % p for p in range(len(sig)-1)]) - def make_coercions(sig): return ';'.join(['p%d = %s' % (p, shared.JS.make_coercion('p%d' % p, sig[p+1], settings)) for p in range(len(sig)-1)]) + ';' - def make_func(name, code, params, coercions): return 'function %s(%s) { %s %s }' % (name, params, coercions, code) - - def make_table(sig, raw): - params = make_params(sig) - coerced_params = make_coerced_params(sig) - coercions = make_coercions(sig) - def make_bad(target=None): - i = Counter.i - Counter.i += 1 - if target is None: target = i - name = 'b' + str(i) - if not settings['ASSERTIONS']: - code = 'abort(%s);' % target - else: - code = 'nullFunc_' + sig + '(%d);' % target - if sig[0] != 'v': - code += 'return %s' % shared.JS.make_initializer(sig[0], settings) + ';' - return name, make_func(name, code, params, coercions) - bad, bad_func = make_bad() # the default bad func - if settings['ASSERTIONS'] <= 1: - Counter.pre = [bad_func] + funcs_js += ['\n// EMSCRIPTEN_END_FUNCS\n'] + + class Counter: + i = 0 + j = 0 + if 'pre' in last_forwarded_json['Functions']['tables']: + pre_tables = last_forwarded_json['Functions']['tables']['pre'] + del last_forwarded_json['Functions']['tables']['pre'] + else: + pre_tables = '' + + def unfloat(s): + return 'd' if s == 'f' else s # lower float to double for ffis + + if settings['ASSERTIONS'] >= 2: + debug_tables = {} + + def make_params(sig): return ','.join(['p%d' % p for p in range(len(sig)-1)]) + def make_coerced_params(sig): return ','.join([shared.JS.make_coercion('p%d', unfloat(sig[p+1]), settings) % p for p in range(len(sig)-1)]) + def make_coercions(sig): return ';'.join(['p%d = %s' % (p, shared.JS.make_coercion('p%d' % p, sig[p+1], settings)) for p in range(len(sig)-1)]) + ';' + def make_func(name, code, params, coercions): return 'function %s(%s) { %s %s }' % (name, params, coercions, code) + + def make_table(sig, raw): + params = make_params(sig) + coerced_params = make_coerced_params(sig) + coercions = make_coercions(sig) + def make_bad(target=None): + i = Counter.i + Counter.i += 1 + if target is None: target = i + name = 'b' + str(i) + if not settings['ASSERTIONS']: + code = 'abort(%s);' % target else: - Counter.pre = [] - start = raw.index('[') - end = raw.rindex(']') - body = raw[start+1:end].split(',') - for j in range(settings['RESERVED_FUNCTION_POINTERS']): - curr = 'jsCall_%s_%s' % (sig, j) - body[settings['FUNCTION_POINTER_ALIGNMENT'] * (1 + j)] = curr - implemented_functions.add(curr) - Counter.j = 0 - def fix_item(item): - j = Counter.j - Counter.j += 1 - newline = Counter.j % 30 == 29 - if item == '0': - if j > 0 and settings['EMULATE_FUNCTION_POINTER_CASTS'] and j in function_pointer_targets: # emulate all non-null pointer calls, if asked to - proper_sig, proper_target = function_pointer_targets[j] - def make_emulated_param(i): - if i >= len(sig): return shared.JS.make_initializer(proper_sig[i], settings) # extra param, just send a zero - return shared.JS.make_coercion('p%d' % (i-1), proper_sig[i], settings, convert_from=sig[i]) - proper_code = proper_target + '(' + ','.join(map(lambda i: make_emulated_param(i+1), range(len(proper_sig)-1))) + ')' - if proper_sig[0] != 'v': - # proper sig has a return, which the wrapper may or may not use - proper_code = shared.JS.make_coercion(proper_code, proper_sig[0], settings) - if sig[0] != 'v': - proper_code = 'return ' + proper_code - else: - # proper sig has no return, we may need a fake return - if sig[0] != 'v': - proper_code = 'return ' + shared.JS.make_initializer(sig[0], settings) - name = 'fpemu_%s_%d' % (sig, j) - wrapper = make_func(name, proper_code, params, coercions) - Counter.pre.append(wrapper) - return name if not newline else (name + '\n') - if settings['ASSERTIONS'] <= 1: - return bad if not newline else (bad + '\n') + code = 'nullFunc_' + sig + '(%d);' % target + if sig[0] != 'v': + code += 'return %s' % shared.JS.make_initializer(sig[0], settings) + ';' + return name, make_func(name, code, params, coercions) + bad, bad_func = make_bad() # the default bad func + if settings['ASSERTIONS'] <= 1: + Counter.pre = [bad_func] + else: + Counter.pre = [] + start = raw.index('[') + end = raw.rindex(']') + body = raw[start+1:end].split(',') + for j in range(settings['RESERVED_FUNCTION_POINTERS']): + curr = 'jsCall_%s_%s' % (sig, j) + body[settings['FUNCTION_POINTER_ALIGNMENT'] * (1 + j)] = curr + implemented_functions.add(curr) + Counter.j = 0 + def fix_item(item): + j = Counter.j + Counter.j += 1 + newline = Counter.j % 30 == 29 + if item == '0': + if j > 0 and settings['EMULATE_FUNCTION_POINTER_CASTS'] and j in function_pointer_targets: # emulate all non-null pointer calls, if asked to + proper_sig, proper_target = function_pointer_targets[j] + def make_emulated_param(i): + if i >= len(sig): return shared.JS.make_initializer(proper_sig[i], settings) # extra param, just send a zero + return shared.JS.make_coercion('p%d' % (i-1), proper_sig[i], settings, convert_from=sig[i]) + proper_code = proper_target + '(' + ','.join(map(lambda i: make_emulated_param(i+1), range(len(proper_sig)-1))) + ')' + if proper_sig[0] != 'v': + # proper sig has a return, which the wrapper may or may not use + proper_code = shared.JS.make_coercion(proper_code, proper_sig[0], settings) + if sig[0] != 'v': + proper_code = 'return ' + proper_code else: - specific_bad, specific_bad_func = make_bad(j) - Counter.pre.append(specific_bad_func) - return specific_bad if not newline else (specific_bad + '\n') - if item not in implemented_functions: - # this is imported into asm, we must wrap it - call_ident = item - if call_ident in metadata['redirects']: call_ident = metadata['redirects'][call_ident] - if not call_ident.startswith('_') and not call_ident.startswith('Math_'): call_ident = '_' + call_ident - code = call_ident + '(' + coerced_params + ')' - if sig[0] != 'v': - # ffis cannot return float - if sig[0] == 'f': code = '+' + code - code = 'return ' + shared.JS.make_coercion(code, sig[0], settings) - code += ';' - Counter.pre.append(make_func(item + '__wrapper', code, params, coercions)) - return item + '__wrapper' - return item if not newline else (item + '\n') - if settings['ASSERTIONS'] >= 2: - debug_tables[sig] = body - body = ','.join(map(fix_item, body)) - return ('\n'.join(Counter.pre), ''.join([raw[:start+1], body, raw[end:]])) - - infos = [make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems()] - Counter.pre = [] - - function_tables_defs = '\n'.join([info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join([info[1] for info in infos]) - - asm_setup = '' - maths = ['Math.' + func for func in ['floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul']] - simdfloattypes = ['float32x4'] - simdinttypes = ['int32x4'] - simdtypes = simdfloattypes + simdinttypes - # TODO: Make mul, min, max, notEqual, lessThanOrEqual, and greaterThanOrEqual - # available for int32x4 too. - simdfuncs = ['add', 'sub', - 'equal', 'lessThan', 'greaterThan', - 'select', 'and', 'or', 'xor', 'not', - 'splat', 'swizzle', 'shuffle', - 'withX', 'withY', 'withZ', 'withW', - 'load', 'store'] - simdfloatfuncs = simdfuncs + ['mul', 'div', 'min', 'max', 'sqrt', - 'fromInt32x4', 'fromInt32x4Bits', - 'notEqual', 'lessThanOrEqual', 'greaterThanOrEqual']; - simdintfuncs = simdfuncs + ['fromFloat32x4', 'fromFloat32x4Bits']; - fundamentals = ['Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array'] - if metadata['simd']: - fundamentals += ['SIMD'] - if settings['ALLOW_MEMORY_GROWTH']: fundamentals.append('byteLength') - math_envs = ['Math.min'] # TODO: move min to maths - - if settings['PRECISE_F32']: maths += ['Math.fround'] - - basic_funcs = ['abort', 'assert'] + [m.replace('.', '_') for m in math_envs] - if settings['RESERVED_FUNCTION_POINTERS'] > 0: basic_funcs.append('jsCall') - if settings['SAFE_HEAP']: basic_funcs += ['SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_FT_MASK'] - if settings['CHECK_HEAP_ALIGN']: basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8'] - if settings['ASSERTIONS']: - if settings['ASSERTIONS'] >= 2: import difflib - for sig in last_forwarded_json['Functions']['tables'].iterkeys(): - basic_funcs += ['nullFunc_' + sig] + # proper sig has no return, we may need a fake return + if sig[0] != 'v': + proper_code = 'return ' + shared.JS.make_initializer(sig[0], settings) + name = 'fpemu_%s_%d' % (sig, j) + wrapper = make_func(name, proper_code, params, coercions) + Counter.pre.append(wrapper) + return name if not newline else (name + '\n') if settings['ASSERTIONS'] <= 1: - extra = ' Module["printErr"]("Build with ASSERTIONS=2 for more info.");' - pointer = ' ' + return bad if not newline else (bad + '\n') else: - pointer = ' \'" + x + "\' ' - asm_setup += '\nvar debug_table_' + sig + ' = ' + json.dumps(debug_tables[sig]) + ';' - extra = ' Module["printErr"]("This pointer might make sense in another type signature: ' - # sort signatures, attempting to show most likely related ones first - sigs = last_forwarded_json['Functions']['tables'].keys() - def keyfunc(other): - ret = 0 - minlen = min(len(other), len(sig)) - maxlen = min(len(other), len(sig)) - if other.startswith(sig) or sig.startswith(other): ret -= 1000 # prioritize prefixes, could be dropped params - ret -= 133*difflib.SequenceMatcher(a=other, b=sig).ratio() # prioritize on diff similarity - ret += 15*abs(len(other) - len(sig))/float(maxlen) # deprioritize the bigger the length difference is - for i in range(minlen): - if other[i] == sig[i]: ret -= 5/float(maxlen) # prioritize on identically-placed params - ret += 20*len(other) # deprioritize on length - return ret - sigs.sort(key=keyfunc) - for other in sigs: - if other != sig: - extra += other + ': " + debug_table_' + other + '[x] + " ' - extra += '"); ' - asm_setup += '\nfunction nullFunc_' + sig + '(x) { Module["printErr"]("Invalid function pointer' + pointer + 'called with signature \'' + sig + '\'. ' + \ - 'Perhaps this is an invalid value (e.g. caused by calling a virtual method on a NULL pointer)? ' + \ - 'Or calling a function with an incorrect type, which will fail? ' + \ - '(it is worth building your source files with -Werror (warnings are errors), as warnings can indicate undefined behavior which can cause this)' + \ - '"); ' + extra + ' abort(x) }\n' - - basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT'] - basic_float_vars = ['NaN', 'Infinity'] - - if metadata.get('preciseI64MathUsed'): - basic_vars += ['cttz_i8', 'ctlz_i8'] - else: - if forwarded_json['Functions']['libraryFunctions'].get('_llvm_cttz_i32'): - basic_vars += ['cttz_i8'] - if forwarded_json['Functions']['libraryFunctions'].get('_llvm_ctlz_i32'): - basic_vars += ['ctlz_i8'] + specific_bad, specific_bad_func = make_bad(j) + Counter.pre.append(specific_bad_func) + return specific_bad if not newline else (specific_bad + '\n') + if item not in implemented_functions: + # this is imported into asm, we must wrap it + call_ident = item + if call_ident in metadata['redirects']: call_ident = metadata['redirects'][call_ident] + if not call_ident.startswith('_') and not call_ident.startswith('Math_'): call_ident = '_' + call_ident + code = call_ident + '(' + coerced_params + ')' + if sig[0] != 'v': + # ffis cannot return float + if sig[0] == 'f': code = '+' + code + code = 'return ' + shared.JS.make_coercion(code, sig[0], settings) + code += ';' + Counter.pre.append(make_func(item + '__wrapper', code, params, coercions)) + return item + '__wrapper' + return item if not newline else (item + '\n') + if settings['ASSERTIONS'] >= 2: + debug_tables[sig] = body + body = ','.join(map(fix_item, body)) + return ('\n'.join(Counter.pre), ''.join([raw[:start+1], body, raw[end:]])) - if settings.get('DLOPEN_SUPPORT'): - for sig in last_forwarded_json['Functions']['tables'].iterkeys(): - basic_vars.append('F_BASE_%s' % sig) - asm_setup += ' var F_BASE_%s = %s;\n' % (sig, 'FUNCTION_TABLE_OFFSET' if settings.get('SIDE_MODULE') else '0') + '\n' + infos = [make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems()] + Counter.pre = [] - asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'setThrew', 'setTempRet0', 'getTempRet0'] + function_tables_defs = '\n'.join([info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join([info[1] for info in infos]) - # See if we need ASYNCIFY functions - # We might not need them even if ASYNCIFY is enabled - need_asyncify = '_emscripten_alloc_async_context' in exported_implemented_functions - if need_asyncify: - basic_vars += ['___async', '___async_unwind', '___async_retval', '___async_cur_frame'] - asm_runtime_funcs += ['setAsync'] + asm_setup = '' + maths = ['Math.' + func for func in ['floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul']] + simdfloattypes = ['float32x4'] + simdinttypes = ['int32x4'] + simdtypes = simdfloattypes + simdinttypes + # TODO: Make mul, min, max, notEqual, lessThanOrEqual, and greaterThanOrEqual + # available for int32x4 too. + simdfuncs = ['add', 'sub', + 'equal', 'lessThan', 'greaterThan', + 'select', 'and', 'or', 'xor', 'not', + 'splat', 'swizzle', 'shuffle', + 'withX', 'withY', 'withZ', 'withW', + 'load', 'store'] + simdfloatfuncs = simdfuncs + ['mul', 'div', 'min', 'max', 'sqrt', + 'fromInt32x4', 'fromInt32x4Bits', + 'notEqual', 'lessThanOrEqual', 'greaterThanOrEqual']; + simdintfuncs = simdfuncs + ['fromFloat32x4', 'fromFloat32x4Bits']; + fundamentals = ['Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array'] + if metadata['simd']: + fundamentals += ['SIMD'] + if settings['ALLOW_MEMORY_GROWTH']: fundamentals.append('byteLength') + math_envs = ['Math.min'] # TODO: move min to maths - # function tables - function_tables = ['dynCall_' + table for table in last_forwarded_json['Functions']['tables']] - function_tables_impls = [] + if settings['PRECISE_F32']: maths += ['Math.fround'] + basic_funcs = ['abort', 'assert'] + [m.replace('.', '_') for m in math_envs] + if settings['RESERVED_FUNCTION_POINTERS'] > 0: basic_funcs.append('jsCall') + if settings['SAFE_HEAP']: basic_funcs += ['SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_FT_MASK'] + if settings['CHECK_HEAP_ALIGN']: basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8'] + if settings['ASSERTIONS']: + if settings['ASSERTIONS'] >= 2: import difflib for sig in last_forwarded_json['Functions']['tables'].iterkeys(): - args = ','.join(['a' + str(i) for i in range(1, len(sig))]) - arg_coercions = ' '.join(['a' + str(i) + '=' + shared.JS.make_coercion('a' + str(i), sig[i], settings) + ';' for i in range(1, len(sig))]) - coerced_args = ','.join([shared.JS.make_coercion('a' + str(i), sig[i], settings) for i in range(1, len(sig))]) - ret = ('return ' if sig[0] != 'v' else '') + shared.JS.make_coercion('FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0], settings) - function_tables_impls.append(''' - function dynCall_%s(index%s%s) { - index = index|0; - %s - %s; - } - ''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret)) - - ffi_args = ','.join([shared.JS.make_coercion('a' + str(i), sig[i], settings, ffi_arg=True) for i in range(1, len(sig))]) - for i in range(settings['RESERVED_FUNCTION_POINTERS']): - jsret = ('return ' if sig[0] != 'v' else '') + shared.JS.make_coercion('jsCall(%d%s%s)' % (i, ',' if ffi_args else '', ffi_args), sig[0], settings, ffi_result=True) - function_tables_impls.append(''' - function jsCall_%s_%s(%s) { - %s - %s; - } - - ''' % (sig, i, args, arg_coercions, jsret)) - shared.Settings.copy(settings) - asm_setup += '\n' + shared.JS.make_invoke(sig) + '\n' - basic_funcs.append('invoke_%s' % sig) - if settings.get('DLOPEN_SUPPORT'): - asm_setup += '\n' + shared.JS.make_extcall(sig) + '\n' - basic_funcs.append('extCall_%s' % sig) - - def quote(prop): - if settings['CLOSURE_COMPILER'] == 2: - return "'" + prop + "'" + basic_funcs += ['nullFunc_' + sig] + if settings['ASSERTIONS'] <= 1: + extra = ' Module["printErr"]("Build with ASSERTIONS=2 for more info.");' + pointer = ' ' else: - return prop + pointer = ' \'" + x + "\' ' + asm_setup += '\nvar debug_table_' + sig + ' = ' + json.dumps(debug_tables[sig]) + ';' + extra = ' Module["printErr"]("This pointer might make sense in another type signature: ' + # sort signatures, attempting to show most likely related ones first + sigs = last_forwarded_json['Functions']['tables'].keys() + def keyfunc(other): + ret = 0 + minlen = min(len(other), len(sig)) + maxlen = min(len(other), len(sig)) + if other.startswith(sig) or sig.startswith(other): ret -= 1000 # prioritize prefixes, could be dropped params + ret -= 133*difflib.SequenceMatcher(a=other, b=sig).ratio() # prioritize on diff similarity + ret += 15*abs(len(other) - len(sig))/float(maxlen) # deprioritize the bigger the length difference is + for i in range(minlen): + if other[i] == sig[i]: ret -= 5/float(maxlen) # prioritize on identically-placed params + ret += 20*len(other) # deprioritize on length + return ret + sigs.sort(key=keyfunc) + for other in sigs: + if other != sig: + extra += other + ': " + debug_table_' + other + '[x] + " ' + extra += '"); ' + asm_setup += '\nfunction nullFunc_' + sig + '(x) { Module["printErr"]("Invalid function pointer' + pointer + 'called with signature \'' + sig + '\'. ' + \ + 'Perhaps this is an invalid value (e.g. caused by calling a virtual method on a NULL pointer)? ' + \ + 'Or calling a function with an incorrect type, which will fail? ' + \ + '(it is worth building your source files with -Werror (warnings are errors), as warnings can indicate undefined behavior which can cause this)' + \ + '"); ' + extra + ' abort(x) }\n' - def access_quote(prop): - if settings['CLOSURE_COMPILER'] == 2: - return "['" + prop + "']" - else: - return '.' + prop - - # calculate exports - exported_implemented_functions = list(exported_implemented_functions) + metadata['initializers'] - exported_implemented_functions.append('runPostSets') - if settings['ALLOW_MEMORY_GROWTH']: - exported_implemented_functions.append('_emscripten_replace_memory') - exports = [] - for export in exported_implemented_functions + asm_runtime_funcs + function_tables: - exports.append(quote(export) + ": " + export) - exports = '{ ' + ', '.join(exports) + ' }' - # calculate globals - try: - del forwarded_json['Variables']['globals']['_llvm_global_ctors'] # not a true variable - except: - pass - # If no named globals, only need externals - global_vars = metadata['externs'] #+ forwarded_json['Variables']['globals'] - global_funcs = list(set([key for key, value in forwarded_json['Functions']['libraryFunctions'].iteritems() if value != 2]).difference(set(global_vars)).difference(implemented_functions)) - def math_fix(g): - return g if not g.startswith('Math_') else g.split('_')[1] - asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global' + access_quote(g) + ';\n' for g in maths]); - asm_global_funcs += ''.join([' var ' + g + '=env' + access_quote(math_fix(g)) + ';\n' for g in basic_funcs + global_funcs]) - if metadata['simd']: - asm_global_funcs += ''.join([' var SIMD_' + ty + '=global' + access_quote('SIMD') + access_quote(ty) + ';\n' for ty in simdtypes]) - asm_global_funcs += ''.join([' var SIMD_' + ty + '_' + g + '=SIMD_' + ty + access_quote(g) + ';\n' for ty in simdinttypes for g in simdintfuncs]) - asm_global_funcs += ''.join([' var SIMD_' + ty + '_' + g + '=SIMD_' + ty + access_quote(g) + ';\n' for ty in simdfloattypes for g in simdfloatfuncs]) - asm_global_vars = ''.join([' var ' + g + '=env' + access_quote(g) + '|0;\n' for g in basic_vars + global_vars]) - # In linkable modules, we need to add some explicit globals for global variables that can be linked and used across modules - if settings.get('MAIN_MODULE') or settings.get('SIDE_MODULE'): - assert settings.get('TARGET_ASMJS_UNKNOWN_EMSCRIPTEN'), 'TODO: support x86 target when linking modules (needs offset of 4 and not 8 here)' - for key, value in forwarded_json['Variables']['globals'].iteritems(): - if value.get('linkable'): - init = forwarded_json['Variables']['indexedGlobals'][key] + 8 # 8 is Runtime.GLOBAL_BASE / STATIC_BASE - if settings.get('SIDE_MODULE'): init = '(H_BASE+' + str(init) + ')|0' - asm_global_vars += ' var %s=%s;\n' % (key, str(init)) - - if settings['POINTER_MASKING']: - for i in [0, 1, 2, 3]: - asm_global_vars += ' var MASK%d=%d;\n' % (i, (settings['TOTAL_MEMORY']-1) & (~((2**i)-1))); - - # sent data - the_global = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in fundamentals]) + ' }' - sending = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars]) + ' }' - # received - receiving = '' - if settings['ASSERTIONS']: - # assert on the runtime being in a valid state when calling into compiled code. The only exceptions are - # some support code like malloc TODO: verify that malloc is actually safe to use that way - receiving = '\n'.join(['var real_' + s + ' = asm["' + s + '"]; asm["' + s + '''"] = function() { - assert(runtimeInitialized, 'you need to wait for the runtime to be ready (e.g. wait for main() to be called)'); - assert(!runtimeExited, 'the runtime was exited (use NO_EXIT_RUNTIME to keep it alive after main() exits)'); - return real_''' + s + '''.apply(null, arguments); + basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT'] + basic_float_vars = ['NaN', 'Infinity'] + + if metadata.get('preciseI64MathUsed'): + basic_vars += ['cttz_i8', 'ctlz_i8'] + else: + if forwarded_json['Functions']['libraryFunctions'].get('_llvm_cttz_i32'): + basic_vars += ['cttz_i8'] + if forwarded_json['Functions']['libraryFunctions'].get('_llvm_ctlz_i32'): + basic_vars += ['ctlz_i8'] + + if settings.get('DLOPEN_SUPPORT'): + for sig in last_forwarded_json['Functions']['tables'].iterkeys(): + basic_vars.append('F_BASE_%s' % sig) + asm_setup += ' var F_BASE_%s = %s;\n' % (sig, 'FUNCTION_TABLE_OFFSET' if settings.get('SIDE_MODULE') else '0') + '\n' + + asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'setThrew', 'setTempRet0', 'getTempRet0'] + + # See if we need ASYNCIFY functions + # We might not need them even if ASYNCIFY is enabled + need_asyncify = '_emscripten_alloc_async_context' in exported_implemented_functions + if need_asyncify: + basic_vars += ['___async', '___async_unwind', '___async_retval', '___async_cur_frame'] + asm_runtime_funcs += ['setAsync'] + + # function tables + function_tables = ['dynCall_' + table for table in last_forwarded_json['Functions']['tables']] + function_tables_impls = [] + + for sig in last_forwarded_json['Functions']['tables'].iterkeys(): + args = ','.join(['a' + str(i) for i in range(1, len(sig))]) + arg_coercions = ' '.join(['a' + str(i) + '=' + shared.JS.make_coercion('a' + str(i), sig[i], settings) + ';' for i in range(1, len(sig))]) + coerced_args = ','.join([shared.JS.make_coercion('a' + str(i), sig[i], settings) for i in range(1, len(sig))]) + ret = ('return ' if sig[0] != 'v' else '') + shared.JS.make_coercion('FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0], settings) + function_tables_impls.append(''' + function dynCall_%s(index%s%s) { + index = index|0; + %s + %s; + } +''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret)) + + ffi_args = ','.join([shared.JS.make_coercion('a' + str(i), sig[i], settings, ffi_arg=True) for i in range(1, len(sig))]) + for i in range(settings['RESERVED_FUNCTION_POINTERS']): + jsret = ('return ' if sig[0] != 'v' else '') + shared.JS.make_coercion('jsCall(%d%s%s)' % (i, ',' if ffi_args else '', ffi_args), sig[0], settings, ffi_result=True) + function_tables_impls.append(''' + function jsCall_%s_%s(%s) { + %s + %s; + } + +''' % (sig, i, args, arg_coercions, jsret)) + shared.Settings.copy(settings) + asm_setup += '\n' + shared.JS.make_invoke(sig) + '\n' + basic_funcs.append('invoke_%s' % sig) + if settings.get('DLOPEN_SUPPORT'): + asm_setup += '\n' + shared.JS.make_extcall(sig) + '\n' + basic_funcs.append('extCall_%s' % sig) + + def quote(prop): + if settings['CLOSURE_COMPILER'] == 2: + return "'" + prop + "'" + else: + return prop + + def access_quote(prop): + if settings['CLOSURE_COMPILER'] == 2: + return "['" + prop + "']" + else: + return '.' + prop + + # calculate exports + exported_implemented_functions = list(exported_implemented_functions) + metadata['initializers'] + exported_implemented_functions.append('runPostSets') + if settings['ALLOW_MEMORY_GROWTH']: + exported_implemented_functions.append('_emscripten_replace_memory') + exports = [] + for export in exported_implemented_functions + asm_runtime_funcs + function_tables: + exports.append(quote(export) + ": " + export) + exports = '{ ' + ', '.join(exports) + ' }' + # calculate globals + try: + del forwarded_json['Variables']['globals']['_llvm_global_ctors'] # not a true variable + except: + pass + # If no named globals, only need externals + global_vars = metadata['externs'] #+ forwarded_json['Variables']['globals'] + global_funcs = list(set([key for key, value in forwarded_json['Functions']['libraryFunctions'].iteritems() if value != 2]).difference(set(global_vars)).difference(implemented_functions)) + def math_fix(g): + return g if not g.startswith('Math_') else g.split('_')[1] + asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global' + access_quote(g) + ';\n' for g in maths]); + asm_global_funcs += ''.join([' var ' + g + '=env' + access_quote(math_fix(g)) + ';\n' for g in basic_funcs + global_funcs]) + if metadata['simd']: + asm_global_funcs += ''.join([' var SIMD_' + ty + '=global' + access_quote('SIMD') + access_quote(ty) + ';\n' for ty in simdtypes]) + asm_global_funcs += ''.join([' var SIMD_' + ty + '_' + g + '=SIMD_' + ty + access_quote(g) + ';\n' for ty in simdinttypes for g in simdintfuncs]) + asm_global_funcs += ''.join([' var SIMD_' + ty + '_' + g + '=SIMD_' + ty + access_quote(g) + ';\n' for ty in simdfloattypes for g in simdfloatfuncs]) + asm_global_vars = ''.join([' var ' + g + '=env' + access_quote(g) + '|0;\n' for g in basic_vars + global_vars]) + # In linkable modules, we need to add some explicit globals for global variables that can be linked and used across modules + if settings.get('MAIN_MODULE') or settings.get('SIDE_MODULE'): + assert settings.get('TARGET_ASMJS_UNKNOWN_EMSCRIPTEN'), 'TODO: support x86 target when linking modules (needs offset of 4 and not 8 here)' + for key, value in forwarded_json['Variables']['globals'].iteritems(): + if value.get('linkable'): + init = forwarded_json['Variables']['indexedGlobals'][key] + 8 # 8 is Runtime.GLOBAL_BASE / STATIC_BASE + if settings.get('SIDE_MODULE'): init = '(H_BASE+' + str(init) + ')|0' + asm_global_vars += ' var %s=%s;\n' % (key, str(init)) + + if settings['POINTER_MASKING']: + for i in [0, 1, 2, 3]: + asm_global_vars += ' var MASK%d=%d;\n' % (i, (settings['TOTAL_MEMORY']-1) & (~((2**i)-1))); + + # sent data + the_global = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in fundamentals]) + ' }' + sending = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars]) + ' }' + # received + receiving = '' + if settings['ASSERTIONS']: + # assert on the runtime being in a valid state when calling into compiled code. The only exceptions are + # some support code like malloc TODO: verify that malloc is actually safe to use that way + receiving = '\n'.join(['var real_' + s + ' = asm["' + s + '"]; asm["' + s + '''"] = function() { +assert(runtimeInitialized, 'you need to wait for the runtime to be ready (e.g. wait for main() to be called)'); +assert(!runtimeExited, 'the runtime was exited (use NO_EXIT_RUNTIME to keep it alive after main() exits)'); +return real_''' + s + '''.apply(null, arguments); }; ''' for s in exported_implemented_functions if s not in ['_malloc', '_free', '_memcpy', '_memset']]) - if not settings['SWAPPABLE_ASM_MODULE']: - receiving += ';\n'.join(['var ' + s + ' = Module["' + s + '"] = asm["' + s + '"]' for s in exported_implemented_functions + function_tables]) - else: - receiving += 'Module["asm"] = asm;\n' + ';\n'.join(['var ' + s + ' = Module["' + s + '"] = function() { return Module["asm"]["' + s + '"].apply(null, arguments) }' for s in exported_implemented_functions + function_tables]) + if not settings['SWAPPABLE_ASM_MODULE']: + receiving += ';\n'.join(['var ' + s + ' = Module["' + s + '"] = asm["' + s + '"]' for s in exported_implemented_functions + function_tables]) + else: + receiving += 'Module["asm"] = asm;\n' + ';\n'.join(['var ' + s + ' = Module["' + s + '"] = function() { return Module["asm"]["' + s + '"].apply(null, arguments) }' for s in exported_implemented_functions + function_tables]) - # finalize + # finalize - if DEBUG: logging.debug('asm text sizes' + str([map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving)])) + if DEBUG: logging.debug('asm text sizes' + str([map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving)])) - funcs_js = [''' + funcs_js = [''' +%s +Module%s = %s; +Module%s = %s; +// EMSCRIPTEN_START_ASM +var asm = (function(global, env, buffer) { %s - Module%s = %s; - Module%s = %s; - // EMSCRIPTEN_START_ASM - var asm = (function(global, env, buffer) { - %s - %s - ''' % (asm_setup, - access_quote('asmGlobalArg'), the_global, - access_quote('asmLibraryArg'), sending, - "'use asm';" if not metadata.get('hasInlineJS') and not settings['SIDE_MODULE'] and settings['ASM_JS'] == 1 else "'almost asm';", ''' - var HEAP8 = new global%s(buffer); - var HEAP16 = new global%s(buffer); - var HEAP32 = new global%s(buffer); - var HEAPU8 = new global%s(buffer); - var HEAPU16 = new global%s(buffer); - var HEAPU32 = new global%s(buffer); - var HEAPF32 = new global%s(buffer); - var HEAPF64 = new global%s(buffer); + %s +''' % (asm_setup, + access_quote('asmGlobalArg'), the_global, + access_quote('asmLibraryArg'), sending, + "'use asm';" if not metadata.get('hasInlineJS') and not settings['SIDE_MODULE'] and settings['ASM_JS'] == 1 else "'almost asm';", ''' + var HEAP8 = new global%s(buffer); + var HEAP16 = new global%s(buffer); + var HEAP32 = new global%s(buffer); + var HEAPU8 = new global%s(buffer); + var HEAPU16 = new global%s(buffer); + var HEAPU32 = new global%s(buffer); + var HEAPF32 = new global%s(buffer); + var HEAPF64 = new global%s(buffer); ''' % (access_quote('Int8Array'), - access_quote('Int16Array'), - access_quote('Int32Array'), - access_quote('Uint8Array'), - access_quote('Uint16Array'), - access_quote('Uint32Array'), - access_quote('Float32Array'), - access_quote('Float64Array')) if not settings['ALLOW_MEMORY_GROWTH'] else ''' - var Int8View = global%s; - var Int16View = global%s; - var Int32View = global%s; - var Uint8View = global%s; - var Uint16View = global%s; - var Uint32View = global%s; - var Float32View = global%s; - var Float64View = global%s; - var HEAP8 = new Int8View(buffer); - var HEAP16 = new Int16View(buffer); - var HEAP32 = new Int32View(buffer); - var HEAPU8 = new Uint8View(buffer); - var HEAPU16 = new Uint16View(buffer); - var HEAPU32 = new Uint32View(buffer); - var HEAPF32 = new Float32View(buffer); - var HEAPF64 = new Float64View(buffer); - var byteLength = global.byteLength; + access_quote('Int16Array'), + access_quote('Int32Array'), + access_quote('Uint8Array'), + access_quote('Uint16Array'), + access_quote('Uint32Array'), + access_quote('Float32Array'), + access_quote('Float64Array')) if not settings['ALLOW_MEMORY_GROWTH'] else ''' + var Int8View = global%s; + var Int16View = global%s; + var Int32View = global%s; + var Uint8View = global%s; + var Uint16View = global%s; + var Uint32View = global%s; + var Float32View = global%s; + var Float64View = global%s; + var HEAP8 = new Int8View(buffer); + var HEAP16 = new Int16View(buffer); + var HEAP32 = new Int32View(buffer); + var HEAPU8 = new Uint8View(buffer); + var HEAPU16 = new Uint16View(buffer); + var HEAPU32 = new Uint32View(buffer); + var HEAPF32 = new Float32View(buffer); + var HEAPF64 = new Float64View(buffer); + var byteLength = global.byteLength; ''' % (access_quote('Int8Array'), - access_quote('Int16Array'), - access_quote('Int32Array'), - access_quote('Uint8Array'), - access_quote('Uint16Array'), - access_quote('Uint32Array'), - access_quote('Float32Array'), - access_quote('Float64Array'))) + '\n' + asm_global_vars + ''' - var __THREW__ = 0; - var threwValue = 0; - var setjmpId = 0; - var undef = 0; - var nan = +env.NaN, inf = +env.Infinity; - var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0; - ''' + ''.join([''' - var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs] + [' var tempFloat = %s;\n' % ('Math_fround(0)' if settings.get('PRECISE_F32') else '0.0')] + ([' const f0 = Math_fround(0);\n'] if settings.get('PRECISE_F32') else []) + ['' if not settings['ALLOW_MEMORY_GROWTH'] else ''' - function _emscripten_replace_memory(newBuffer) { - if ((byteLength(newBuffer) & 0xffffff || byteLength(newBuffer) <= 0xffffff) || byteLength(newBuffer) > 0x80000000) return false; - HEAP8 = new Int8View(newBuffer); - HEAP16 = new Int16View(newBuffer); - HEAP32 = new Int32View(newBuffer); - HEAPU8 = new Uint8View(newBuffer); - HEAPU16 = new Uint16View(newBuffer); - HEAPU32 = new Uint32View(newBuffer); - HEAPF32 = new Float32View(newBuffer); - HEAPF64 = new Float64View(newBuffer); - buffer = newBuffer; - return true; - } + access_quote('Int16Array'), + access_quote('Int32Array'), + access_quote('Uint8Array'), + access_quote('Uint16Array'), + access_quote('Uint32Array'), + access_quote('Float32Array'), + access_quote('Float64Array'))) + '\n' + asm_global_vars + ''' + var __THREW__ = 0; + var threwValue = 0; + var setjmpId = 0; + var undef = 0; + var nan = +env.NaN, inf = +env.Infinity; + var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0; +''' + ''.join([''' + var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs] + [' var tempFloat = %s;\n' % ('Math_fround(0)' if settings.get('PRECISE_F32') else '0.0')] + ([' const f0 = Math_fround(0);\n'] if settings.get('PRECISE_F32') else []) + ['' if not settings['ALLOW_MEMORY_GROWTH'] else ''' +function _emscripten_replace_memory(newBuffer) { + if ((byteLength(newBuffer) & 0xffffff || byteLength(newBuffer) <= 0xffffff) || byteLength(newBuffer) > 0x80000000) return false; + HEAP8 = new Int8View(newBuffer); + HEAP16 = new Int16View(newBuffer); + HEAP32 = new Int32View(newBuffer); + HEAPU8 = new Uint8View(newBuffer); + HEAPU16 = new Uint16View(newBuffer); + HEAPU32 = new Uint32View(newBuffer); + HEAPF32 = new Float32View(newBuffer); + HEAPF64 = new Float64View(newBuffer); + buffer = newBuffer; + return true; +} '''] + [''' - // EMSCRIPTEN_START_FUNCS - function stackAlloc(size) { - size = size|0; - var ret = 0; - ret = STACKTOP; - STACKTOP = (STACKTOP + size)|0; - ''' + ('STACKTOP = (STACKTOP + 3)&-4;' if settings['TARGET_X86'] else 'STACKTOP = (STACKTOP + 15)&-16;\n') + - ('if ((STACKTOP|0) >= (STACK_MAX|0)) abort();\n' if settings['ASSERTIONS'] else '') + ''' - return ret|0; - } - function stackSave() { - return STACKTOP|0; - } - function stackRestore(top) { - top = top|0; - STACKTOP = top; - } +// EMSCRIPTEN_START_FUNCS +function stackAlloc(size) { + size = size|0; + var ret = 0; + ret = STACKTOP; + STACKTOP = (STACKTOP + size)|0; +''' + ('STACKTOP = (STACKTOP + 3)&-4;' if settings['TARGET_X86'] else 'STACKTOP = (STACKTOP + 15)&-16;\n') + + ('if ((STACKTOP|0) >= (STACK_MAX|0)) abort();\n' if settings['ASSERTIONS'] else '') + ''' + return ret|0; +} +function stackSave() { + return STACKTOP|0; +} +function stackRestore(top) { + top = top|0; + STACKTOP = top; +} ''' + (''' - function setAsync() { - ___async = 1; - }''' if need_asyncify else '') + ''' - function setThrew(threw, value) { - threw = threw|0; - value = value|0; - if ((__THREW__|0) == 0) { - __THREW__ = threw; - threwValue = value; - } - } - function copyTempFloat(ptr) { - ptr = ptr|0; - HEAP8[tempDoublePtr>>0] = HEAP8[ptr>>0]; - HEAP8[tempDoublePtr+1>>0] = HEAP8[ptr+1>>0]; - HEAP8[tempDoublePtr+2>>0] = HEAP8[ptr+2>>0]; - HEAP8[tempDoublePtr+3>>0] = HEAP8[ptr+3>>0]; - } - function copyTempDouble(ptr) { - ptr = ptr|0; - HEAP8[tempDoublePtr>>0] = HEAP8[ptr>>0]; - HEAP8[tempDoublePtr+1>>0] = HEAP8[ptr+1>>0]; - HEAP8[tempDoublePtr+2>>0] = HEAP8[ptr+2>>0]; - HEAP8[tempDoublePtr+3>>0] = HEAP8[ptr+3>>0]; - HEAP8[tempDoublePtr+4>>0] = HEAP8[ptr+4>>0]; - HEAP8[tempDoublePtr+5>>0] = HEAP8[ptr+5>>0]; - HEAP8[tempDoublePtr+6>>0] = HEAP8[ptr+6>>0]; - HEAP8[tempDoublePtr+7>>0] = HEAP8[ptr+7>>0]; - } - function setTempRet0(value) { - value = value|0; - tempRet0 = value; - } - function getTempRet0() { - return tempRet0|0; +function setAsync() { + ___async = 1; +}''' if need_asyncify else '') + ''' +function setThrew(threw, value) { + threw = threw|0; + value = value|0; + if ((__THREW__|0) == 0) { + __THREW__ = threw; + threwValue = value; } - '''] + funcs_js + [''' - %s +} +function copyTempFloat(ptr) { + ptr = ptr|0; + HEAP8[tempDoublePtr>>0] = HEAP8[ptr>>0]; + HEAP8[tempDoublePtr+1>>0] = HEAP8[ptr+1>>0]; + HEAP8[tempDoublePtr+2>>0] = HEAP8[ptr+2>>0]; + HEAP8[tempDoublePtr+3>>0] = HEAP8[ptr+3>>0]; +} +function copyTempDouble(ptr) { + ptr = ptr|0; + HEAP8[tempDoublePtr>>0] = HEAP8[ptr>>0]; + HEAP8[tempDoublePtr+1>>0] = HEAP8[ptr+1>>0]; + HEAP8[tempDoublePtr+2>>0] = HEAP8[ptr+2>>0]; + HEAP8[tempDoublePtr+3>>0] = HEAP8[ptr+3>>0]; + HEAP8[tempDoublePtr+4>>0] = HEAP8[ptr+4>>0]; + HEAP8[tempDoublePtr+5>>0] = HEAP8[ptr+5>>0]; + HEAP8[tempDoublePtr+6>>0] = HEAP8[ptr+6>>0]; + HEAP8[tempDoublePtr+7>>0] = HEAP8[ptr+7>>0]; +} +function setTempRet0(value) { + value = value|0; + tempRet0 = value; +} +function getTempRet0() { + return tempRet0|0; +} +'''] + funcs_js + [''' + %s - return %s; - }) - // EMSCRIPTEN_END_ASM - (%s, %s, buffer); - %s; - ''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n '), exports, - 'Module' + access_quote('asmGlobalArg'), - 'Module' + access_quote('asmLibraryArg'), - receiving)] - - if not settings.get('SIDE_MODULE'): - funcs_js.append(''' - Runtime.stackAlloc = asm['stackAlloc']; - Runtime.stackSave = asm['stackSave']; - Runtime.stackRestore = asm['stackRestore']; - Runtime.setTempRet0 = asm['setTempRet0']; - Runtime.getTempRet0 = asm['getTempRet0']; - ''') + return %s; +}) +// EMSCRIPTEN_END_ASM +(%s, %s, buffer); +%s; +''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n '), exports, + 'Module' + access_quote('asmGlobalArg'), + 'Module' + access_quote('asmLibraryArg'), + receiving)] - # Set function table masks - masks = {} - max_mask = 0 - for sig, table in last_forwarded_json['Functions']['tables'].iteritems(): - mask = table.count(',') - masks[sig] = str(mask) - max_mask = max(mask, max_mask) - def function_table_maskize(js, masks): - def fix(m): - sig = m.groups(0)[0] - return masks[sig] - return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]] - funcs_js = map(lambda js: function_table_maskize(js, masks), funcs_js) + if not settings.get('SIDE_MODULE'): + funcs_js.append(''' +Runtime.stackAlloc = asm['stackAlloc']; +Runtime.stackSave = asm['stackSave']; +Runtime.stackRestore = asm['stackRestore']; +Runtime.setTempRet0 = asm['setTempRet0']; +Runtime.getTempRet0 = asm['getTempRet0']; +''') - if settings.get('DLOPEN_SUPPORT'): - funcs_js.append(''' - asm.maxFunctionIndex = %(max_mask)d; - DLFCN.registerFunctions(asm, %(max_mask)d+1, %(sigs)s, Module); - Module.SYMBOL_TABLE = SYMBOL_TABLE; - ''' % { 'max_mask': max_mask, 'sigs': str(map(str, last_forwarded_json['Functions']['tables'].keys())) }) + # Set function table masks + masks = {} + max_mask = 0 + for sig, table in last_forwarded_json['Functions']['tables'].iteritems(): + mask = table.count(',') + masks[sig] = str(mask) + max_mask = max(mask, max_mask) + def function_table_maskize(js, masks): + def fix(m): + sig = m.groups(0)[0] + return masks[sig] + return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]] + funcs_js = map(lambda js: function_table_maskize(js, masks), funcs_js) - else: - function_tables_defs = '\n'.join([table for table in last_forwarded_json['Functions']['tables'].itervalues()]) - outfile.write(function_tables_defs) - funcs_js = [''' - // EMSCRIPTEN_START_FUNCS - '''] + funcs_js + [''' - // EMSCRIPTEN_END_FUNCS - '''] + if settings.get('DLOPEN_SUPPORT'): + funcs_js.append(''' + asm.maxFunctionIndex = %(max_mask)d; + DLFCN.registerFunctions(asm, %(max_mask)d+1, %(sigs)s, Module); + Module.SYMBOL_TABLE = SYMBOL_TABLE; +''' % { 'max_mask': max_mask, 'sigs': str(map(str, last_forwarded_json['Functions']['tables'].keys())) }) # Create symbol table for self-dlopen if settings.get('DLOPEN_SUPPORT'): From 166d6efaa51fd2978bac03a65dace7c10f0119c2 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Nov 2014 16:36:51 -0800 Subject: [PATCH 139/161] EXPORT_FUNCTION_TABLES option --- emscripten.py | 10 ++++++++++ src/settings.js | 2 ++ 2 files changed, 12 insertions(+) diff --git a/emscripten.py b/emscripten.py index 594da72b41b1b..e311bb55a907f 100755 --- a/emscripten.py +++ b/emscripten.py @@ -935,6 +935,11 @@ def save_settings(): all_exported_functions = set(settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise for additional_export in settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented all_exported_functions.add('_' + additional_export) + if settings['EXPORT_FUNCTION_TABLES']: + for table in last_forwarded_json['Functions']['tables'].values(): + for func in table.split('[')[1].split(']')[0].split(','): + if func[0] == '_': + all_exported_functions.add(func) exported_implemented_functions = set(metadata['exports']) export_bindings = settings['EXPORT_BINDINGS'] export_all = settings['EXPORT_ALL'] @@ -1280,6 +1285,11 @@ def math_fix(g): else: receiving += 'Module["asm"] = asm;\n' + ';\n'.join(['var ' + s + ' = Module["' + s + '"] = function() { return Module["asm"]["' + s + '"].apply(null, arguments) }' for s in exported_implemented_functions + function_tables]) + if settings['EXPORT_FUNCTION_TABLES']: + receiving += '\n' + for table in last_forwarded_json['Functions']['tables'].values(): + receiving += table + '\n' + # finalize if DEBUG: logging.debug('asm text sizes' + str([map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving)])) diff --git a/src/settings.js b/src/settings.js index aeeaf27fc07d4..4ec83eac810ec 100644 --- a/src/settings.js +++ b/src/settings.js @@ -397,6 +397,8 @@ var EXPORT_ALL = 0; // If true, we export all the symbols. Note that this does * // still eliminate functions as dead. This just exports them on the Module object. var EXPORT_BINDINGS = 0; // Export all bindings generator functions (prefixed with emscripten_bind_). This // is necessary to use the WebIDL binder or bindings generator with asm.js +var EXPORT_FUNCTION_TABLES = 0; // If true, export all the functions appearing in a function table, and the + // tables themselves. var RETAIN_COMPILER_SETTINGS = 0; // Remembers the values of these settings, and makes them accessible // through Runtime.getCompilerSetting and emscripten_get_compiler_setting. // To see what is retained, look for compilerSettings in the generated code. From 7e3f14f74942ae48500cb677831e3c2f27084859 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Nov 2014 16:57:10 -0800 Subject: [PATCH 140/161] enable native optimizer on minified output --- tools/js_optimizer.py | 2 +- tools/optimizer/optimizer.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index ce93cb349866a..d0abb3a9f7f8f 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -9,7 +9,7 @@ def path_from_root(*pathelems): return os.path.join(__rootpath__, *pathelems) -NATIVE_PASSES = set(['asm', 'asmPreciseF32', 'receiveJSON', 'emitJSON', 'eliminate', 'eliminateMemSafe', 'simplifyExpressions', 'simplifyIfs', 'optimizeFrounds', 'registerize', 'minifyNames', 'minifyLocals']) +NATIVE_PASSES = set(['asm', 'asmPreciseF32', 'receiveJSON', 'emitJSON', 'eliminate', 'eliminateMemSafe', 'simplifyExpressions', 'simplifyIfs', 'optimizeFrounds', 'registerize', 'minifyNames', 'minifyLocals', 'minifyWhitespace']) JS_OPTIMIZER = path_from_root('tools', 'js-optimizer.js') diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 6f4c8e0fa2887..9911c3d603e63 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -2788,6 +2788,7 @@ int main(int argc, char **argv) { else if (str == "simplifyIfs") simplifyIfs(doc); else if (str == "registerize") registerize(doc); else if (str == "minifyLocals") minifyLocals(doc); + else if (str == "minifyWhitespace") {} else { fprintf(stderr, "unrecognized argument: %s\n", str.c_str()); assert(0); From cb6b842e7a0fd0f81b726872316ec819ad0e0c0f Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Nov 2014 16:57:31 -0800 Subject: [PATCH 141/161] add test for native optimizer --- tests/test_other.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_other.py b/tests/test_other.py index c356e04892c65..33ff4353dd387 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -4407,3 +4407,18 @@ def test_require(self): output = Popen(NODE_JS + ['-e', 'require("./a.out.js")'], stdout=PIPE, stderr=PIPE).communicate() assert output == ('hello, world!\n \n', ''), 'expected no output, got\n===\nSTDOUT\n%s\n===\nSTDERR\n%s\n===\n' % output + def test_native_optimizer(self): + old_debug = os.environ.get('EMCC_DEBUG') + old_native = os.environ.get('EMCC_NATIVE_OPTIMIZER') + try: + os.environ['EMCC_DEBUG'] = '1' + os.environ['EMCC_NATIVE_OPTIMIZER'] = '1' + out, err = Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-O2'], stderr=PIPE).communicate() + finally: + if old_debug: os.environ['EMCC_DEBUG'] = old_debug + else: del os.environ['EMCC_DEBUG'] + if old_native: os.environ['EMCC_NATIVE_OPTIMIZER'] = old_native + else: del os.environ['EMCC_NATIVE_OPTIMIZER'] + self.assertContained('js optimizer using native', err) + self.assertContained('hello, world!', run_js('a.out.js')) + From a563da7adcebe324aa00da2d9be9e3e71cd4f634 Mon Sep 17 00:00:00 2001 From: Chad Austin Date: Fri, 14 Nov 2014 21:41:53 -0800 Subject: [PATCH 142/161] replace the Function.prototype.bind in embind asm.js with a bind-time-generated trampoline. This results in a 500% reduction in call overhead in Chrome. (Very little change in Firefox) --- src/embind/embind.js | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/embind/embind.js b/src/embind/embind.js index 36486e6009a7a..02ad388c710e7 100644 --- a/src/embind/embind.js +++ b/src/embind/embind.js @@ -901,6 +901,21 @@ var LibraryEmbind = { $requireFunction__deps: ['$readLatin1String', '$throwBindingError'], $requireFunction: function(signature, rawFunction) { signature = readLatin1String(signature); + + function makeDynCaller(dynCall) { + var args = []; + for (var i = 1; i < signature.length; ++i) { + args.push('a' + i); + } + + var name = 'dynCall_' + signature + '_' + rawFunction; + var body = 'return function ' + name + '(' + args.join(', ') + ') {\n'; + body += ' return dynCall(rawFunction' + (args.length ? ', ' : '') + args.join(', ') + ');\n'; + body += '};\n'; + + return (new Function('dynCall', 'rawFunction', body))(dynCall, rawFunction); + } + var fp; // asm.js does not define FUNCTION_TABLE if (typeof FUNCTION_TABLE === "undefined") { @@ -913,9 +928,6 @@ var LibraryEmbind = { // This has three main penalties: // - dynCall is another function call in the path from JavaScript to C++. // - JITs may not predict through the function table indirection at runtime. - // - Function.prototype.bind generally benchmarks poorly relative to - // function objects, but using 'arguments' would confound JITs and - // possibly allocate. var dc = asm['dynCall_' + signature]; if (dc === undefined) { // We will always enter this branch if the signature @@ -927,7 +939,7 @@ var LibraryEmbind = { throwBindingError("No dynCall invoker for signature: " + signature); } } - fp = dc.bind(undefined, rawFunction); + fp = makeDynCaller(dc); } else { fp = FUNCTION_TABLE[rawFunction]; } From 5a3ae46c88dad44fdf89bed25e85a5f8085735fe Mon Sep 17 00:00:00 2001 From: Chad Austin Date: Fri, 14 Nov 2014 22:43:46 -0800 Subject: [PATCH 143/161] Allow embind to benefit from the EXPORT_FUNCTION_TABLES option. --- emscripten.py | 2 ++ src/embind/embind.js | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/emscripten.py b/emscripten.py index e311bb55a907f..2b04a0398e114 100755 --- a/emscripten.py +++ b/emscripten.py @@ -1288,6 +1288,8 @@ def math_fix(g): if settings['EXPORT_FUNCTION_TABLES']: receiving += '\n' for table in last_forwarded_json['Functions']['tables'].values(): + tableName = table.split()[1] + table = table.replace('var ' + tableName, 'var ' + tableName + ' = Module["' + tableName + '"]') receiving += table + '\n' # finalize diff --git a/src/embind/embind.js b/src/embind/embind.js index 02ad388c710e7..393e97d7469b2 100644 --- a/src/embind/embind.js +++ b/src/embind/embind.js @@ -917,8 +917,11 @@ var LibraryEmbind = { } var fp; - // asm.js does not define FUNCTION_TABLE - if (typeof FUNCTION_TABLE === "undefined") { + if (Module['FUNCTION_TABLE_' + signature] !== undefined) { + fp = Module['FUNCTION_TABLE_' + signature][rawFunction]; + } else if (typeof FUNCTION_TABLE !== "undefined") { + fp = FUNCTION_TABLE[rawFunction]; + } else { // asm.js does not give direct access to the function tables, // and thus we must go through the dynCall interface which allows // calling into a signature's function table by pointer value. @@ -940,8 +943,6 @@ var LibraryEmbind = { } } fp = makeDynCaller(dc); - } else { - fp = FUNCTION_TABLE[rawFunction]; } if (typeof fp !== "function") { From bdf1cb4270c80ad09dcb1556adf1647c116c0f2f Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Sat, 15 Nov 2014 14:58:49 +0700 Subject: [PATCH 144/161] Fix typo. --- src/library_gl.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library_gl.js b/src/library_gl.js index 798fec905908f..662018d25745d 100644 --- a/src/library_gl.js +++ b/src/library_gl.js @@ -76,7 +76,7 @@ var LibraryGL = { GL.lastError = errorCode; } }, - // Get a new ID for a texture/buffer/etc., while keeping the table dense and fast. Creation is farely rare so it is worth optimizing lookups later. + // Get a new ID for a texture/buffer/etc., while keeping the table dense and fast. Creation is fairly rare so it is worth optimizing lookups later. getNewId: function(table) { var ret = GL.counter++; for (var i = table.length; i < ret; i++) { From 0dd7995d9681af42e360f944e22bd9dbe890a1ab Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sat, 15 Nov 2014 09:28:51 -0800 Subject: [PATCH 145/161] more error logging in other.test_native_optimizer --- tests/test_other.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_other.py b/tests/test_other.py index 33ff4353dd387..9571515e1b4af 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -4420,5 +4420,6 @@ def test_native_optimizer(self): if old_native: os.environ['EMCC_NATIVE_OPTIMIZER'] = old_native else: del os.environ['EMCC_NATIVE_OPTIMIZER'] self.assertContained('js optimizer using native', err) + assert os.path.exists('a.out.js'), err self.assertContained('hello, world!', run_js('a.out.js')) From 1205ecc101ba39a472a39a5cd5341aacbdca9fef Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sat, 15 Nov 2014 16:50:52 -0800 Subject: [PATCH 146/161] avoid using sys/stat.h in optimizer --- tools/optimizer/optimizer.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 9911c3d603e63..64a3d694899a7 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -1,6 +1,6 @@ -#include #include #include +#include #include @@ -2749,14 +2749,12 @@ void minifyLocals(Ref ast) { int main(int argc, char **argv) { // Read input file - char *input = argv[1]; - struct stat st; - int result = stat(input, &st); - assert(result == 0); - int size = st.st_size; - //printf("reading %s, %d bytes\n", input, size); + FILE *f = fopen(argv[1], "r"); + assert(f); + fseek(f, 0, SEEK_END); + int size = ftell(f); char *json = new char[size+1]; - FILE *f = fopen(input, "rb"); + rewind(f); int num = fread(json, 1, size, f); assert(num == size); fclose(f); From a5d95d90c6e3c882858d0acd6ed7bb9f1c22101d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sat, 15 Nov 2014 22:42:53 -0800 Subject: [PATCH 147/161] move some included headers in optimizer from c to c++ --- tools/optimizer/optimizer.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 64a3d694899a7..72969f16bd65c 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -1,7 +1,6 @@ -#include -#include -#include - +#include +#include +#include #include #include "minijson.h" From 37f69c0e691ae9f700c594b37902059d3ce781b6 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sun, 16 Nov 2014 10:35:08 -0800 Subject: [PATCH 148/161] disable other.test_native_optimizer as the optimizer does not build on at least 2/3 of the bots --- tests/test_other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_other.py b/tests/test_other.py index 9571515e1b4af..c8c146cf90776 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -4407,7 +4407,7 @@ def test_require(self): output = Popen(NODE_JS + ['-e', 'require("./a.out.js")'], stdout=PIPE, stderr=PIPE).communicate() assert output == ('hello, world!\n \n', ''), 'expected no output, got\n===\nSTDOUT\n%s\n===\nSTDERR\n%s\n===\n' % output - def test_native_optimizer(self): + def zzztest_native_optimizer(self): old_debug = os.environ.get('EMCC_DEBUG') old_native = os.environ.get('EMCC_NATIVE_OPTIMIZER') try: From c4904a8c63b1649ba643c1eb17f75906dcc18923 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sun, 16 Nov 2014 16:46:34 -0800 Subject: [PATCH 149/161] try to use a system compiler if using our clang fails to build the native optimizer; fixes #2997 --- tests/test_other.py | 2 +- tools/js_optimizer.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/test_other.py b/tests/test_other.py index c8c146cf90776..9571515e1b4af 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -4407,7 +4407,7 @@ def test_require(self): output = Popen(NODE_JS + ['-e', 'require("./a.out.js")'], stdout=PIPE, stderr=PIPE).communicate() assert output == ('hello, world!\n \n', ''), 'expected no output, got\n===\nSTDOUT\n%s\n===\nSTDERR\n%s\n===\n' % output - def zzztest_native_optimizer(self): + def test_native_optimizer(self): old_debug = os.environ.get('EMCC_DEBUG') old_native = os.environ.get('EMCC_NATIVE_OPTIMIZER') try: diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index d0abb3a9f7f8f..2d8abbd7eae45 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -32,9 +32,14 @@ def create_optimizer(): shared.logging.debug('building native optimizer') output = shared.Cache.get_path('optimizer.exe') shared.try_delete(output) - subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-O3', '-std=c++11', '-fno-exceptions', '-fno-rtti', '-o', output]).communicate() # , '-g', '-fno-omit-frame-pointer' - assert os.path.exists(output) - return output + errs = [] + for compiler in [shared.CLANG, 'g++', 'clang++']: # try our clang first, otherwise hope for a system compiler in the path + shared.logging.debug(' using ' + compiler) + out, err = subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-O3', '-std=c++11', '-fno-exceptions', '-fno-rtti', '-o', output], stderr=subprocess.PIPE).communicate() + # for profiling/debugging: '-g', '-fno-omit-frame-pointer' + if os.path.exists(output): return output + errs.append(err) + raise Exception('failed to build native optimizer, errors from each attempt: ' + '\n=================\n'.join(errs)) return shared.Cache.get('optimizer.exe', create_optimizer, extension='exe') # Check if we should run a pass or set of passes natively. if a set of passes, they must all be valid to run in the native optimizer at once. From 2145cc9d7aa2624562c10aaf80efb8f253ea0e5a Mon Sep 17 00:00:00 2001 From: Aleksander Guryanov Date: Mon, 17 Nov 2014 23:24:34 +0700 Subject: [PATCH 150/161] Add implementation for TTF_GlyphMetrics --- src/library_sdl.js | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/library_sdl.js b/src/library_sdl.js index a008bd68ec1d6..f2b99e560c2c0 100644 --- a/src/library_sdl.js +++ b/src/library_sdl.js @@ -2927,6 +2927,27 @@ var LibrarySDL = { return 0; }, + TTF_GlyphMetrics: function(font, ch, minx, maxx, miny, maxy, advance) { + var fontData = SDL.fonts[font]; + var width = SDL.estimateTextWidth(fontData, String.fromCharCode(ch)); + + if (advance) { + {{{ makeSetValue('advance', '0', 'width', 'i32') }}}; + } + if (minx) { + {{{ makeSetValue('minx', '0', '0', 'i32') }}}; + } + if (maxx) { + {{{ makeSetValue('maxx', '0', 'width', 'i32') }}}; + } + if (miny) { + {{{ makeSetValue('miny', '0', '0', 'i32') }}}; + } + if (maxy) { + {{{ makeSetValue('maxy', '0', 'fontData.size', 'i32') }}}; + } + }, + TTF_FontAscent: function(font) { var fontData = SDL.fonts[font]; return (fontData.size*0.98)|0; // XXX From c21e7f05b75abe275f73dc7020be5d3d706ffc40 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 17 Nov 2014 09:40:15 -0800 Subject: [PATCH 151/161] actually try different compilers when building optimizer; #2997 --- tools/js_optimizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 2d8abbd7eae45..553a749a34adb 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -35,7 +35,7 @@ def create_optimizer(): errs = [] for compiler in [shared.CLANG, 'g++', 'clang++']: # try our clang first, otherwise hope for a system compiler in the path shared.logging.debug(' using ' + compiler) - out, err = subprocess.Popen([shared.CLANG, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-O3', '-std=c++11', '-fno-exceptions', '-fno-rtti', '-o', output], stderr=subprocess.PIPE).communicate() + out, err = subprocess.Popen([compiler, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-O3', '-std=c++11', '-fno-exceptions', '-fno-rtti', '-o', output], stderr=subprocess.PIPE).communicate() # for profiling/debugging: '-g', '-fno-omit-frame-pointer' if os.path.exists(output): return output errs.append(err) From 8217f16d1e8cb91240ea5c80896ab6a4895faf49 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 17 Nov 2014 09:56:15 -0800 Subject: [PATCH 152/161] fix glfw time type; fixes #3002 --- src/library_glfw.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library_glfw.js b/src/library_glfw.js index 00300e583f7e8..a2f13c3cb4312 100644 --- a/src/library_glfw.js +++ b/src/library_glfw.js @@ -704,7 +704,7 @@ var LibraryGLFW = { glfwInit: function() { if (GLFW.windows) return 1; // GL_TRUE - GLFW.initalTime = GLFW.getTime(); + GLFW.initialTime = GLFW.getTime(); GLFW.hints = GLFW.defaultHints; GLFW.windows = new Array() GLFW.active = null; From f514007c8b492ecf1c2adae1074322966686787e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 17 Nov 2014 10:47:56 -0800 Subject: [PATCH 153/161] fix EMCONFIGURE_JS on object outputs; fixes #2994 --- emcc | 17 ++++++++--------- tests/test_other.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/emcc b/emcc index 17c8753a9b2f5..ae2ad7dddaab5 100755 --- a/emcc +++ b/emcc @@ -247,10 +247,8 @@ if CONFIGURE_CONFIG or CMAKE_CONFIG: cmd[i+1] += '.js' target = cmd[i+1] break - print 't1', target if not target: target = 'a.out.js' - print 't2', target, only_object os.environ['EMMAKEN_JUST_CONFIGURE_RECURSE'] = '1' ret = subprocess.call(cmd) os.environ['EMMAKEN_JUST_CONFIGURE_RECURSE'] = '' @@ -258,13 +256,14 @@ if CONFIGURE_CONFIG or CMAKE_CONFIG: if target.endswith('.js'): shutil.copyfile(target, target[:-3]) target = target[:-3] - src = open(target).read() - full_node = ' '.join(shared.NODE_JS) - if os.path.sep not in full_node: - full_node = '/usr/bin/' + full_node # TODO: use whereis etc. And how about non-*NIX? - open(target, 'w').write('#!' + full_node + '\n' + src) # add shebang - import stat - os.chmod(target, stat.S_IMODE(os.stat(target).st_mode) | stat.S_IXUSR) # make executable + if not target.endswith(BITCODE_ENDINGS): + src = open(target).read() + full_node = ' '.join(shared.NODE_JS) + if os.path.sep not in full_node: + full_node = '/usr/bin/' + full_node # TODO: use whereis etc. And how about non-*NIX? + open(target, 'w').write('#!' + full_node + '\n' + src) # add shebang + import stat + os.chmod(target, stat.S_IMODE(os.stat(target).st_mode) | stat.S_IXUSR) # make executable exit(ret) if os.environ.get('EMMAKEN_COMPILER'): diff --git a/tests/test_other.py b/tests/test_other.py index 9571515e1b4af..4bcafa70de0a3 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -4423,3 +4423,13 @@ def test_native_optimizer(self): assert os.path.exists('a.out.js'), err self.assertContained('hello, world!', run_js('a.out.js')) + def test_emconfigure_js_o(self): + # issue 2994 + try: + os.environ['EMCONFIGURE_JS'] = '1' + Popen([PYTHON, path_from_root('emconfigure'), EMCC, '-c', '-o', 'a.o', path_from_root('tests', 'hello_world.c')]).communicate() + Popen([PYTHON, EMCC, 'a.o']).communicate() + assert 'hello, world!' in run_js(self.in_dir('a.out.js')) + finally: + del os.environ['EMCONFIGURE_JS'] + From 8aacee1e06f6decf1fdc793e9ce374bab533be74 Mon Sep 17 00:00:00 2001 From: Akira Takahashi Date: Wed, 19 Nov 2014 00:52:07 +0900 Subject: [PATCH 154/161] add missing semicolon. s/ChangeLog/ChangeLog.markdown/ Correct url is follow: --- site/source/docs/introducing_emscripten/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/source/docs/introducing_emscripten/release_notes.rst b/site/source/docs/introducing_emscripten/release_notes.rst index cfe7df5d72098..b43cc5ad226a8 100644 --- a/site/source/docs/introducing_emscripten/release_notes.rst +++ b/site/source/docs/introducing_emscripten/release_notes.rst @@ -14,7 +14,7 @@ ChangeLog ========= -The ChangeLog for Emscripten |release| (|today|) is listed below (master version `here `_). +The ChangeLog for Emscripten |release| (|today|) is listed below (master version `here `_). .. include:: ../../../../ChangeLog :literal: From 929ccfb5e431e566e96fa74dc4685a202b206f82 Mon Sep 17 00:00:00 2001 From: Akira Takahashi Date: Wed, 19 Nov 2014 00:52:15 +0900 Subject: [PATCH 155/161] add author --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 50410d975a39d..123ce0cc70ad6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -169,4 +169,5 @@ a license to everyone to use it as detailed in LICENSE.) * Dan Glastonbury (copyright owned by Mozilla Foundation) * Warren Seine (copyright owned by Aerys SAS) * Petr Babicka +* Akira Takahashi From 8a70c782a3afdfae49b08c9ad807691dcf4101b6 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 18 Nov 2014 10:27:43 -0800 Subject: [PATCH 156/161] use python to run emcc in other.test_emconfigure_js_o, to fix it on windows --- tests/test_other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_other.py b/tests/test_other.py index 4bcafa70de0a3..0f69b3d952c44 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -4427,7 +4427,7 @@ def test_emconfigure_js_o(self): # issue 2994 try: os.environ['EMCONFIGURE_JS'] = '1' - Popen([PYTHON, path_from_root('emconfigure'), EMCC, '-c', '-o', 'a.o', path_from_root('tests', 'hello_world.c')]).communicate() + Popen([PYTHON, path_from_root('emconfigure'), PYTHON, EMCC, '-c', '-o', 'a.o', path_from_root('tests', 'hello_world.c')]).communicate() Popen([PYTHON, EMCC, 'a.o']).communicate() assert 'hello, world!' in run_js(self.in_dir('a.out.js')) finally: From 1362b205884307198c82a74503946de6e110293e Mon Sep 17 00:00:00 2001 From: Jukka Jylanki Date: Tue, 18 Nov 2014 20:58:33 +0200 Subject: [PATCH 157/161] Fix another missing 'python emcc' vs 'emcc' for Windows in other.test_emconfigure_js_o. --- emcc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/emcc b/emcc index ae2ad7dddaab5..8911f9adc0303 100755 --- a/emcc +++ b/emcc @@ -229,7 +229,9 @@ if CONFIGURE_CONFIG or CMAKE_CONFIG: yield el idx += 1 - cmd = [compiler] + list(filter_emscripten_options(sys.argv[1:])) + if compiler == shared.EMCC: compiler = [shared.PYTHON, shared.EMCC] + else: compiler = [compiler] + cmd = compiler + list(filter_emscripten_options(sys.argv[1:])) if not use_js: cmd += shared.EMSDK_OPTS + ['-D__EMSCRIPTEN__', '-DEMSCRIPTEN'] if use_js: cmd += ['-s', 'ERROR_ON_UNDEFINED_SYMBOLS=1'] # configure tests should fail when an undefined symbol exists From 7bc0b6c6c89542323ac8ed25af8660cfb5d73beb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 20 Nov 2014 00:30:06 +0200 Subject: [PATCH 158/161] Fix interactive.test_html5_fullscreen --- tests/test_interactive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_interactive.py b/tests/test_interactive.py index 3e567059ac8eb..067def270aef7 100644 --- a/tests/test_interactive.py +++ b/tests/test_interactive.py @@ -20,7 +20,7 @@ def setUpClass(self): print def test_html5_fullscreen(self): - self.btest(path_from_root('tests', 'test_html5_fullscreen.c'), expected='0', args=['-s', 'EXPORTED_FUNCTIONS=["_requestFullscreen","_enterSoftFullscreen","_main"]']) + self.btest(path_from_root('tests', 'test_html5_fullscreen.c'), expected='0', args=['-s', 'EXPORTED_FUNCTIONS=["_requestFullscreen","_enterSoftFullscreen","_main"]', '--shell-file', path_from_root('tests', 'test_html5_fullscreen.html')]) def test_html5_mouse(self): self.btest(path_from_root('tests', 'test_html5_mouse.c'), expected='0') From 0a9fd833216547b095338e3523ea1960b0c0b465 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 19 Nov 2014 14:35:03 -0800 Subject: [PATCH 159/161] make interactive.test_sdl_audio_panning run a more reasonable amount of time --- tests/sdl_audio_panning.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sdl_audio_panning.c b/tests/sdl_audio_panning.c index c6bf38270a86a..a137637ff6fc7 100644 --- a/tests/sdl_audio_panning.c +++ b/tests/sdl_audio_panning.c @@ -27,7 +27,7 @@ void pan() { int panning = Mix_SetPanning(channel, left, right); assert(panning != 0); - if (frames > 30 * 100) + if (frames > 30 * 10) done(); } From 3df099da587f78b84e9417dfdedb5404f1c4116a Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 19 Nov 2014 14:53:54 -0800 Subject: [PATCH 160/161] disable webrtc test and add warning --- src/settings.js | 2 +- tests/test_sockets.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/settings.js b/src/settings.js index 4ec83eac810ec..8cf44149e5d3b 100644 --- a/src/settings.js +++ b/src/settings.js @@ -254,7 +254,7 @@ var LIBRARY_DEBUG = 0; // Print out when we enter a library call (library*.js). // want it back. A simple way to set it in C++ is // emscripten_run_script("Runtime.debug = ...;"); var SOCKET_DEBUG = 0; // Log out socket/network data transfer. -var SOCKET_WEBRTC = 0; // Select socket backend, either webrtc or websockets. +var SOCKET_WEBRTC = 0; // Select socket backend, either webrtc or websockets. XXX webrtc is not currently tested, may be broken // As well as being configurable at compile time via the "-s" option the WEBSOCKET_URL and WEBSOCKET_SUBPROTOCOL // settings may configured at run time via the Module object e.g. diff --git a/tests/test_sockets.py b/tests/test_sockets.py index f311ed8910398..ce6b1ab3d1bf1 100644 --- a/tests/test_sockets.py +++ b/tests/test_sockets.py @@ -365,7 +365,7 @@ def test_enet(self): # finally: # clean_pids(pids); - def test_webrtc(self): + def zzztest_webrtc(self): # XXX see src/settings.js, this is disabled pending investigation host_src = 'webrtc_host.c' peer_src = 'webrtc_peer.c' From 805ea30dc019c70f4598e8611b6728e337fbed59 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 19 Nov 2014 15:10:36 -0800 Subject: [PATCH 161/161] 1.27.0 --- emscripten-version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emscripten-version.txt b/emscripten-version.txt index 9f67c6b08c1c4..1227f05ac63eb 100644 --- a/emscripten-version.txt +++ b/emscripten-version.txt @@ -1,2 +1,2 @@ -1.26.1 +1.27.0