diff options
Diffstat (limited to 'src/tree_sitter/unicode')
| -rw-r--r-- | src/tree_sitter/unicode/ICU_SHA | 1 | ||||
| -rw-r--r-- | src/tree_sitter/unicode/LICENSE | 414 | ||||
| -rw-r--r-- | src/tree_sitter/unicode/README.md | 29 | ||||
| -rw-r--r-- | src/tree_sitter/unicode/ptypes.h | 1 | ||||
| -rw-r--r-- | src/tree_sitter/unicode/umachine.h | 448 | ||||
| -rw-r--r-- | src/tree_sitter/unicode/urename.h | 1 | ||||
| -rw-r--r-- | src/tree_sitter/unicode/utf.h | 1 | ||||
| -rw-r--r-- | src/tree_sitter/unicode/utf16.h | 733 | ||||
| -rw-r--r-- | src/tree_sitter/unicode/utf8.h | 881 |
9 files changed, 0 insertions, 2509 deletions
diff --git a/src/tree_sitter/unicode/ICU_SHA b/src/tree_sitter/unicode/ICU_SHA deleted file mode 100644 index 3622283ba3..0000000000 --- a/src/tree_sitter/unicode/ICU_SHA +++ /dev/null @@ -1 +0,0 @@ -552b01f61127d30d6589aa4bf99468224979b661 diff --git a/src/tree_sitter/unicode/LICENSE b/src/tree_sitter/unicode/LICENSE deleted file mode 100644 index 2e01e36876..0000000000 --- a/src/tree_sitter/unicode/LICENSE +++ /dev/null @@ -1,414 +0,0 @@ -COPYRIGHT AND PERMISSION NOTICE (ICU 58 and later) - -Copyright © 1991-2019 Unicode, Inc. All rights reserved. -Distributed under the Terms of Use in https://www.unicode.org/copyright.html. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of the Unicode data files and any associated documentation -(the "Data Files") or Unicode software and any associated documentation -(the "Software") to deal in the Data Files or Software -without restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, and/or sell copies of -the Data Files or Software, and to permit persons to whom the Data Files -or Software are furnished to do so, provided that either -(a) this copyright and permission notice appear with all copies -of the Data Files or Software, or -(b) this copyright and permission notice appear in associated -Documentation. - -THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT OF THIRD PARTY RIGHTS. -IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS -NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL -DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, -DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THE DATA FILES OR SOFTWARE. - -Except as contained in this notice, the name of a copyright holder -shall not be used in advertising or otherwise to promote the sale, -use or other dealings in these Data Files or Software without prior -written authorization of the copyright holder. - ---------------------- - -Third-Party Software Licenses - -This section contains third-party software notices and/or additional -terms for licensed third-party software components included within ICU -libraries. - -1. ICU License - ICU 1.8.1 to ICU 57.1 - -COPYRIGHT AND PERMISSION NOTICE - -Copyright (c) 1995-2016 International Business Machines Corporation and others -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, and/or sell copies of the Software, and to permit persons -to whom the Software is furnished to do so, provided that the above -copyright notice(s) and this permission notice appear in all copies of -the Software and that both the above copyright notice(s) and this -permission notice appear in supporting documentation. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT -OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY -SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER -RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -Except as contained in this notice, the name of a copyright holder -shall not be used in advertising or otherwise to promote the sale, use -or other dealings in this Software without prior written authorization -of the copyright holder. - -All trademarks and registered trademarks mentioned herein are the -property of their respective owners. - -2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt) - - # The Google Chrome software developed by Google is licensed under - # the BSD license. Other software included in this distribution is - # provided under other licenses, as set forth below. - # - # The BSD License - # http://opensource.org/licenses/bsd-license.php - # Copyright (C) 2006-2008, Google Inc. - # - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without - # modification, are permitted provided that the following conditions are met: - # - # Redistributions of source code must retain the above copyright notice, - # this list of conditions and the following disclaimer. - # Redistributions in binary form must reproduce the above - # copyright notice, this list of conditions and the following - # disclaimer in the documentation and/or other materials provided with - # the distribution. - # Neither the name of Google Inc. nor the names of its - # contributors may be used to endorse or promote products derived from - # this software without specific prior written permission. - # - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, - # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - # - # - # The word list in cjdict.txt are generated by combining three word lists - # listed below with further processing for compound word breaking. The - # frequency is generated with an iterative training against Google web - # corpora. - # - # * Libtabe (Chinese) - # - https://sourceforge.net/project/?group_id=1519 - # - Its license terms and conditions are shown below. - # - # * IPADIC (Japanese) - # - http://chasen.aist-nara.ac.jp/chasen/distribution.html - # - Its license terms and conditions are shown below. - # - # ---------COPYING.libtabe ---- BEGIN-------------------- - # - # /* - # * Copyright (c) 1999 TaBE Project. - # * Copyright (c) 1999 Pai-Hsiang Hsiao. - # * All rights reserved. - # * - # * Redistribution and use in source and binary forms, with or without - # * modification, are permitted provided that the following conditions - # * are met: - # * - # * . Redistributions of source code must retain the above copyright - # * notice, this list of conditions and the following disclaimer. - # * . Redistributions in binary form must reproduce the above copyright - # * notice, this list of conditions and the following disclaimer in - # * the documentation and/or other materials provided with the - # * distribution. - # * . Neither the name of the TaBE Project nor the names of its - # * contributors may be used to endorse or promote products derived - # * from this software without specific prior written permission. - # * - # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - # * OF THE POSSIBILITY OF SUCH DAMAGE. - # */ - # - # /* - # * Copyright (c) 1999 Computer Systems and Communication Lab, - # * Institute of Information Science, Academia - # * Sinica. All rights reserved. - # * - # * Redistribution and use in source and binary forms, with or without - # * modification, are permitted provided that the following conditions - # * are met: - # * - # * . Redistributions of source code must retain the above copyright - # * notice, this list of conditions and the following disclaimer. - # * . Redistributions in binary form must reproduce the above copyright - # * notice, this list of conditions and the following disclaimer in - # * the documentation and/or other materials provided with the - # * distribution. - # * . Neither the name of the Computer Systems and Communication Lab - # * nor the names of its contributors may be used to endorse or - # * promote products derived from this software without specific - # * prior written permission. - # * - # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - # * OF THE POSSIBILITY OF SUCH DAMAGE. - # */ - # - # Copyright 1996 Chih-Hao Tsai @ Beckman Institute, - # University of Illinois - # c-tsai4@uiuc.edu http://casper.beckman.uiuc.edu/~c-tsai4 - # - # ---------------COPYING.libtabe-----END-------------------------------- - # - # - # ---------------COPYING.ipadic-----BEGIN------------------------------- - # - # Copyright 2000, 2001, 2002, 2003 Nara Institute of Science - # and Technology. All Rights Reserved. - # - # Use, reproduction, and distribution of this software is permitted. - # Any copy of this software, whether in its original form or modified, - # must include both the above copyright notice and the following - # paragraphs. - # - # Nara Institute of Science and Technology (NAIST), - # the copyright holders, disclaims all warranties with regard to this - # software, including all implied warranties of merchantability and - # fitness, in no event shall NAIST be liable for - # any special, indirect or consequential damages or any damages - # whatsoever resulting from loss of use, data or profits, whether in an - # action of contract, negligence or other tortuous action, arising out - # of or in connection with the use or performance of this software. - # - # A large portion of the dictionary entries - # originate from ICOT Free Software. The following conditions for ICOT - # Free Software applies to the current dictionary as well. - # - # Each User may also freely distribute the Program, whether in its - # original form or modified, to any third party or parties, PROVIDED - # that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear - # on, or be attached to, the Program, which is distributed substantially - # in the same form as set out herein and that such intended - # distribution, if actually made, will neither violate or otherwise - # contravene any of the laws and regulations of the countries having - # jurisdiction over the User or the intended distribution itself. - # - # NO WARRANTY - # - # The program was produced on an experimental basis in the course of the - # research and development conducted during the project and is provided - # to users as so produced on an experimental basis. Accordingly, the - # program is provided without any warranty whatsoever, whether express, - # implied, statutory or otherwise. The term "warranty" used herein - # includes, but is not limited to, any warranty of the quality, - # performance, merchantability and fitness for a particular purpose of - # the program and the nonexistence of any infringement or violation of - # any right of any third party. - # - # Each user of the program will agree and understand, and be deemed to - # have agreed and understood, that there is no warranty whatsoever for - # the program and, accordingly, the entire risk arising from or - # otherwise connected with the program is assumed by the user. - # - # Therefore, neither ICOT, the copyright holder, or any other - # organization that participated in or was otherwise related to the - # development of the program and their respective officials, directors, - # officers and other employees shall be held liable for any and all - # damages, including, without limitation, general, special, incidental - # and consequential damages, arising out of or otherwise in connection - # with the use or inability to use the program or any product, material - # or result produced or otherwise obtained by using the program, - # regardless of whether they have been advised of, or otherwise had - # knowledge of, the possibility of such damages at any time during the - # project or thereafter. Each user will be deemed to have agreed to the - # foregoing by his or her commencement of use of the program. The term - # "use" as used herein includes, but is not limited to, the use, - # modification, copying and distribution of the program and the - # production of secondary products from the program. - # - # In the case where the program, whether in its original form or - # modified, was distributed or delivered to or received by a user from - # any person, organization or entity other than ICOT, unless it makes or - # grants independently of ICOT any specific warranty to the user in - # writing, such person, organization or entity, will also be exempted - # from and not be held liable to the user for any such damages as noted - # above as far as the program is concerned. - # - # ---------------COPYING.ipadic-----END---------------------------------- - -3. Lao Word Break Dictionary Data (laodict.txt) - - # Copyright (c) 2013 International Business Machines Corporation - # and others. All Rights Reserved. - # - # Project: http://code.google.com/p/lao-dictionary/ - # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt - # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt - # (copied below) - # - # This file is derived from the above dictionary, with slight - # modifications. - # ---------------------------------------------------------------------- - # Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell. - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without - # modification, - # are permitted provided that the following conditions are met: - # - # - # Redistributions of source code must retain the above copyright notice, this - # list of conditions and the following disclaimer. Redistributions in - # binary form must reproduce the above copyright notice, this list of - # conditions and the following disclaimer in the documentation and/or - # other materials provided with the distribution. - # - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - # OF THE POSSIBILITY OF SUCH DAMAGE. - # -------------------------------------------------------------------------- - -4. Burmese Word Break Dictionary Data (burmesedict.txt) - - # Copyright (c) 2014 International Business Machines Corporation - # and others. All Rights Reserved. - # - # This list is part of a project hosted at: - # github.com/kanyawtech/myanmar-karen-word-lists - # - # -------------------------------------------------------------------------- - # Copyright (c) 2013, LeRoy Benjamin Sharon - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without - # modification, are permitted provided that the following conditions - # are met: Redistributions of source code must retain the above - # copyright notice, this list of conditions and the following - # disclaimer. Redistributions in binary form must reproduce the - # above copyright notice, this list of conditions and the following - # disclaimer in the documentation and/or other materials provided - # with the distribution. - # - # Neither the name Myanmar Karen Word Lists, nor the names of its - # contributors may be used to endorse or promote products derived - # from this software without specific prior written permission. - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, - # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS - # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR - # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF - # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - # SUCH DAMAGE. - # -------------------------------------------------------------------------- - -5. Time Zone Database - - ICU uses the public domain data and code derived from Time Zone -Database for its time zone support. The ownership of the TZ database -is explained in BCP 175: Procedure for Maintaining the Time Zone -Database section 7. - - # 7. Database Ownership - # - # The TZ database itself is not an IETF Contribution or an IETF - # document. Rather it is a pre-existing and regularly updated work - # that is in the public domain, and is intended to remain in the - # public domain. Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do - # not apply to the TZ Database or contributions that individuals make - # to it. Should any claims be made and substantiated against the TZ - # Database, the organization that is providing the IANA - # Considerations defined in this RFC, under the memorandum of - # understanding with the IETF, currently ICANN, may act in accordance - # with all competent court orders. No ownership claims will be made - # by ICANN or the IETF Trust on the database or the code. Any person - # making a contribution to the database or code waives all rights to - # future claims in that contribution or in the TZ Database. - -6. Google double-conversion - -Copyright 2006-2011, the V8 project authors. All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - * Neither the name of Google Inc. nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/tree_sitter/unicode/README.md b/src/tree_sitter/unicode/README.md deleted file mode 100644 index 623b8e3843..0000000000 --- a/src/tree_sitter/unicode/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# ICU Parts - -This directory contains a small subset of files from the Unicode organization's [ICU repository](https://github.com/unicode-org/icu). - -### License - -The license for these files is contained in the `LICENSE` file within this directory. - -### Contents - -* Source files taken from the [`icu4c/source/common/unicode`](https://github.com/unicode-org/icu/tree/552b01f61127d30d6589aa4bf99468224979b661/icu4c/source/common/unicode) directory: - * `utf8.h` - * `utf16.h` - * `umachine.h` -* Empty source files that are referenced by the above source files, but whose original contents in `libicu` are not needed: - * `ptypes.h` - * `urename.h` - * `utf.h` -* `ICU_SHA` - File containing the Git SHA of the commit in the `icu` repository from which the files were obtained. -* `LICENSE` - The license file from the [`icu4c`](https://github.com/unicode-org/icu/tree/552b01f61127d30d6589aa4bf99468224979b661/icu4c) directory of the `icu` repository. -* `README.md` - This text file. - -### Updating ICU - -To incorporate changes from the upstream `icu` repository: - -* Update `ICU_SHA` with the new Git SHA. -* Update `LICENSE` with the license text from the directory mentioned above. -* Update `utf8.h`, `utf16.h`, and `umachine.h` with their new contents in the `icu` repository. diff --git a/src/tree_sitter/unicode/ptypes.h b/src/tree_sitter/unicode/ptypes.h deleted file mode 100644 index ac79ad0f98..0000000000 --- a/src/tree_sitter/unicode/ptypes.h +++ /dev/null @@ -1 +0,0 @@ -// This file must exist in order for `utf8.h` and `utf16.h` to be used. diff --git a/src/tree_sitter/unicode/umachine.h b/src/tree_sitter/unicode/umachine.h deleted file mode 100644 index bbf6ef9c8b..0000000000 --- a/src/tree_sitter/unicode/umachine.h +++ /dev/null @@ -1,448 +0,0 @@ -// © 2016 and later: Unicode, Inc. and others. -// License & terms of use: http://www.unicode.org/copyright.html -/* -****************************************************************************** -* -* Copyright (C) 1999-2015, International Business Machines -* Corporation and others. All Rights Reserved. -* -****************************************************************************** -* file name: umachine.h -* encoding: UTF-8 -* tab size: 8 (not used) -* indentation:4 -* -* created on: 1999sep13 -* created by: Markus W. Scherer -* -* This file defines basic types and constants for ICU to be -* platform-independent. umachine.h and utf.h are included into -* utypes.h to provide all the general definitions for ICU. -* All of these definitions used to be in utypes.h before -* the UTF-handling macros made this unmaintainable. -*/ - -#ifndef __UMACHINE_H__ -#define __UMACHINE_H__ - - -/** - * \file - * \brief Basic types and constants for UTF - * - * <h2> Basic types and constants for UTF </h2> - * This file defines basic types and constants for utf.h to be - * platform-independent. umachine.h and utf.h are included into - * utypes.h to provide all the general definitions for ICU. - * All of these definitions used to be in utypes.h before - * the UTF-handling macros made this unmaintainable. - * - */ -/*==========================================================================*/ -/* Include platform-dependent definitions */ -/* which are contained in the platform-specific file platform.h */ -/*==========================================================================*/ - -#include "./ptypes.h" /* platform.h is included in ptypes.h */ - -/* - * ANSI C headers: - * stddef.h defines wchar_t - */ -#include <stddef.h> - -/*==========================================================================*/ -/* For C wrappers, we use the symbol U_STABLE. */ -/* This works properly if the includer is C or C++. */ -/* Functions are declared U_STABLE return-type U_EXPORT2 function-name()... */ -/*==========================================================================*/ - -/** - * \def U_CFUNC - * This is used in a declaration of a library private ICU C function. - * @stable ICU 2.4 - */ - -/** - * \def U_CDECL_BEGIN - * This is used to begin a declaration of a library private ICU C API. - * @stable ICU 2.4 - */ - -/** - * \def U_CDECL_END - * This is used to end a declaration of a library private ICU C API - * @stable ICU 2.4 - */ - -#ifdef __cplusplus -# define U_CFUNC extern "C" -# define U_CDECL_BEGIN extern "C" { -# define U_CDECL_END } -#else -# define U_CFUNC extern -# define U_CDECL_BEGIN -# define U_CDECL_END -#endif - -#ifndef U_ATTRIBUTE_DEPRECATED -/** - * \def U_ATTRIBUTE_DEPRECATED - * This is used for GCC specific attributes - * @internal - */ -#if U_GCC_MAJOR_MINOR >= 302 -# define U_ATTRIBUTE_DEPRECATED __attribute__ ((deprecated)) -/** - * \def U_ATTRIBUTE_DEPRECATED - * This is used for Visual C++ specific attributes - * @internal - */ -#elif defined(_MSC_VER) && (_MSC_VER >= 1400) -# define U_ATTRIBUTE_DEPRECATED __declspec(deprecated) -#else -# define U_ATTRIBUTE_DEPRECATED -#endif -#endif - -/** This is used to declare a function as a public ICU C API @stable ICU 2.0*/ -#define U_CAPI U_CFUNC U_EXPORT -/** This is used to declare a function as a stable public ICU C API*/ -#define U_STABLE U_CAPI -/** This is used to declare a function as a draft public ICU C API */ -#define U_DRAFT U_CAPI -/** This is used to declare a function as a deprecated public ICU C API */ -#define U_DEPRECATED U_CAPI U_ATTRIBUTE_DEPRECATED -/** This is used to declare a function as an obsolete public ICU C API */ -#define U_OBSOLETE U_CAPI -/** This is used to declare a function as an internal ICU C API */ -#define U_INTERNAL U_CAPI - -/** - * \def U_OVERRIDE - * Defined to the C++11 "override" keyword if available. - * Denotes a class or member which is an override of the base class. - * May result in an error if it applied to something not an override. - * @internal - */ -#ifndef U_OVERRIDE -#define U_OVERRIDE override -#endif - -/** - * \def U_FINAL - * Defined to the C++11 "final" keyword if available. - * Denotes a class or member which may not be overridden in subclasses. - * May result in an error if subclasses attempt to override. - * @internal - */ -#if !defined(U_FINAL) || defined(U_IN_DOXYGEN) -#define U_FINAL final -#endif - -// Before ICU 65, function-like, multi-statement ICU macros were just defined as -// series of statements wrapped in { } blocks and the caller could choose to -// either treat them as if they were actual functions and end the invocation -// with a trailing ; creating an empty statement after the block or else omit -// this trailing ; using the knowledge that the macro would expand to { }. -// -// But doing so doesn't work well with macros that look like functions and -// compiler warnings about empty statements (ICU-20601) and ICU 65 therefore -// switches to the standard solution of wrapping such macros in do { } while. -// -// This will however break existing code that depends on being able to invoke -// these macros without a trailing ; so to be able to remain compatible with -// such code the wrapper is itself defined as macros so that it's possible to -// build ICU 65 and later with the old macro behaviour, like this: -// -// CPPFLAGS='-DUPRV_BLOCK_MACRO_BEGIN="" -DUPRV_BLOCK_MACRO_END=""' -// runConfigureICU ... - -/** - * \def UPRV_BLOCK_MACRO_BEGIN - * Defined as the "do" keyword by default. - * @internal - */ -#ifndef UPRV_BLOCK_MACRO_BEGIN -#define UPRV_BLOCK_MACRO_BEGIN do -#endif - -/** - * \def UPRV_BLOCK_MACRO_END - * Defined as "while (FALSE)" by default. - * @internal - */ -#ifndef UPRV_BLOCK_MACRO_END -#define UPRV_BLOCK_MACRO_END while (FALSE) -#endif - -/*==========================================================================*/ -/* limits for int32_t etc., like in POSIX inttypes.h */ -/*==========================================================================*/ - -#ifndef INT8_MIN -/** The smallest value an 8 bit signed integer can hold @stable ICU 2.0 */ -# define INT8_MIN ((int8_t)(-128)) -#endif -#ifndef INT16_MIN -/** The smallest value a 16 bit signed integer can hold @stable ICU 2.0 */ -# define INT16_MIN ((int16_t)(-32767-1)) -#endif -#ifndef INT32_MIN -/** The smallest value a 32 bit signed integer can hold @stable ICU 2.0 */ -# define INT32_MIN ((int32_t)(-2147483647-1)) -#endif - -#ifndef INT8_MAX -/** The largest value an 8 bit signed integer can hold @stable ICU 2.0 */ -# define INT8_MAX ((int8_t)(127)) -#endif -#ifndef INT16_MAX -/** The largest value a 16 bit signed integer can hold @stable ICU 2.0 */ -# define INT16_MAX ((int16_t)(32767)) -#endif -#ifndef INT32_MAX -/** The largest value a 32 bit signed integer can hold @stable ICU 2.0 */ -# define INT32_MAX ((int32_t)(2147483647)) -#endif - -#ifndef UINT8_MAX -/** The largest value an 8 bit unsigned integer can hold @stable ICU 2.0 */ -# define UINT8_MAX ((uint8_t)(255U)) -#endif -#ifndef UINT16_MAX -/** The largest value a 16 bit unsigned integer can hold @stable ICU 2.0 */ -# define UINT16_MAX ((uint16_t)(65535U)) -#endif -#ifndef UINT32_MAX -/** The largest value a 32 bit unsigned integer can hold @stable ICU 2.0 */ -# define UINT32_MAX ((uint32_t)(4294967295U)) -#endif - -#if defined(U_INT64_T_UNAVAILABLE) -# error int64_t is required for decimal format and rule-based number format. -#else -# ifndef INT64_C -/** - * Provides a platform independent way to specify a signed 64-bit integer constant. - * note: may be wrong for some 64 bit platforms - ensure your compiler provides INT64_C - * @stable ICU 2.8 - */ -# define INT64_C(c) c ## LL -# endif -# ifndef UINT64_C -/** - * Provides a platform independent way to specify an unsigned 64-bit integer constant. - * note: may be wrong for some 64 bit platforms - ensure your compiler provides UINT64_C - * @stable ICU 2.8 - */ -# define UINT64_C(c) c ## ULL -# endif -# ifndef U_INT64_MIN -/** The smallest value a 64 bit signed integer can hold @stable ICU 2.8 */ -# define U_INT64_MIN ((int64_t)(INT64_C(-9223372036854775807)-1)) -# endif -# ifndef U_INT64_MAX -/** The largest value a 64 bit signed integer can hold @stable ICU 2.8 */ -# define U_INT64_MAX ((int64_t)(INT64_C(9223372036854775807))) -# endif -# ifndef U_UINT64_MAX -/** The largest value a 64 bit unsigned integer can hold @stable ICU 2.8 */ -# define U_UINT64_MAX ((uint64_t)(UINT64_C(18446744073709551615))) -# endif -#endif - -/*==========================================================================*/ -/* Boolean data type */ -/*==========================================================================*/ - -/** The ICU boolean type @stable ICU 2.0 */ -typedef int8_t UBool; - -#ifndef TRUE -/** The TRUE value of a UBool @stable ICU 2.0 */ -# define TRUE 1 -#endif -#ifndef FALSE -/** The FALSE value of a UBool @stable ICU 2.0 */ -# define FALSE 0 -#endif - - -/*==========================================================================*/ -/* Unicode data types */ -/*==========================================================================*/ - -/* wchar_t-related definitions -------------------------------------------- */ - -/* - * \def U_WCHAR_IS_UTF16 - * Defined if wchar_t uses UTF-16. - * - * @stable ICU 2.0 - */ -/* - * \def U_WCHAR_IS_UTF32 - * Defined if wchar_t uses UTF-32. - * - * @stable ICU 2.0 - */ -#if !defined(U_WCHAR_IS_UTF16) && !defined(U_WCHAR_IS_UTF32) -# ifdef __STDC_ISO_10646__ -# if (U_SIZEOF_WCHAR_T==2) -# define U_WCHAR_IS_UTF16 -# elif (U_SIZEOF_WCHAR_T==4) -# define U_WCHAR_IS_UTF32 -# endif -# elif defined __UCS2__ -# if (U_PF_OS390 <= U_PLATFORM && U_PLATFORM <= U_PF_OS400) && (U_SIZEOF_WCHAR_T==2) -# define U_WCHAR_IS_UTF16 -# endif -# elif defined(__UCS4__) || (U_PLATFORM == U_PF_OS400 && defined(__UTF32__)) -# if (U_SIZEOF_WCHAR_T==4) -# define U_WCHAR_IS_UTF32 -# endif -# elif U_PLATFORM_IS_DARWIN_BASED || (U_SIZEOF_WCHAR_T==4 && U_PLATFORM_IS_LINUX_BASED) -# define U_WCHAR_IS_UTF32 -# elif U_PLATFORM_HAS_WIN32_API -# define U_WCHAR_IS_UTF16 -# endif -#endif - -/* UChar and UChar32 definitions -------------------------------------------- */ - -/** Number of bytes in a UChar. @stable ICU 2.0 */ -#define U_SIZEOF_UCHAR 2 - -/** - * \def U_CHAR16_IS_TYPEDEF - * If 1, then char16_t is a typedef and not a real type (yet) - * @internal - */ -#if (U_PLATFORM == U_PF_AIX) && defined(__cplusplus) &&(U_CPLUSPLUS_VERSION < 11) -// for AIX, uchar.h needs to be included -# include <uchar.h> -# define U_CHAR16_IS_TYPEDEF 1 -#elif defined(_MSC_VER) && (_MSC_VER < 1900) -// Versions of Visual Studio/MSVC below 2015 do not support char16_t as a real type, -// and instead use a typedef. https://msdn.microsoft.com/library/bb531344.aspx -# define U_CHAR16_IS_TYPEDEF 1 -#else -# define U_CHAR16_IS_TYPEDEF 0 -#endif - - -/** - * \var UChar - * - * The base type for UTF-16 code units and pointers. - * Unsigned 16-bit integer. - * Starting with ICU 59, C++ API uses char16_t directly, while C API continues to use UChar. - * - * UChar is configurable by defining the macro UCHAR_TYPE - * on the preprocessor or compiler command line: - * -DUCHAR_TYPE=uint16_t or -DUCHAR_TYPE=wchar_t (if U_SIZEOF_WCHAR_T==2) etc. - * (The UCHAR_TYPE can also be \#defined earlier in this file, for outside the ICU library code.) - * This is for transitional use from application code that uses uint16_t or wchar_t for UTF-16. - * - * The default is UChar=char16_t. - * - * C++11 defines char16_t as bit-compatible with uint16_t, but as a distinct type. - * - * In C, char16_t is a simple typedef of uint_least16_t. - * ICU requires uint_least16_t=uint16_t for data memory mapping. - * On macOS, char16_t is not available because the uchar.h standard header is missing. - * - * @stable ICU 4.4 - */ - -#if 1 - // #if 1 is normal. UChar defaults to char16_t in C++. - // For configuration testing of UChar=uint16_t temporarily change this to #if 0. - // The intltest Makefile #defines UCHAR_TYPE=char16_t, - // so we only #define it to uint16_t if it is undefined so far. -#elif !defined(UCHAR_TYPE) -# define UCHAR_TYPE uint16_t -#endif - -#if defined(U_COMBINED_IMPLEMENTATION) || defined(U_COMMON_IMPLEMENTATION) || \ - defined(U_I18N_IMPLEMENTATION) || defined(U_IO_IMPLEMENTATION) - // Inside the ICU library code, never configurable. - typedef char16_t UChar; -#elif defined(UCHAR_TYPE) - typedef UCHAR_TYPE UChar; -#elif defined(__cplusplus) - typedef char16_t UChar; -#else - typedef uint16_t UChar; -#endif - -/** - * \var OldUChar - * Default ICU 58 definition of UChar. - * A base type for UTF-16 code units and pointers. - * Unsigned 16-bit integer. - * - * Define OldUChar to be wchar_t if that is 16 bits wide. - * If wchar_t is not 16 bits wide, then define UChar to be uint16_t. - * - * This makes the definition of OldUChar platform-dependent - * but allows direct string type compatibility with platforms with - * 16-bit wchar_t types. - * - * This is how UChar was defined in ICU 58, for transition convenience. - * Exception: ICU 58 UChar was defined to UCHAR_TYPE if that macro was defined. - * The current UChar responds to UCHAR_TYPE but OldUChar does not. - * - * @stable ICU 59 - */ -#if U_SIZEOF_WCHAR_T==2 - typedef wchar_t OldUChar; -#elif defined(__CHAR16_TYPE__) - typedef __CHAR16_TYPE__ OldUChar; -#else - typedef uint16_t OldUChar; -#endif - -/** - * Define UChar32 as a type for single Unicode code points. - * UChar32 is a signed 32-bit integer (same as int32_t). - * - * The Unicode code point range is 0..0x10ffff. - * All other values (negative or >=0x110000) are illegal as Unicode code points. - * They may be used as sentinel values to indicate "done", "error" - * or similar non-code point conditions. - * - * Before ICU 2.4 (Jitterbug 2146), UChar32 was defined - * to be wchar_t if that is 32 bits wide (wchar_t may be signed or unsigned) - * or else to be uint32_t. - * That is, the definition of UChar32 was platform-dependent. - * - * @see U_SENTINEL - * @stable ICU 2.4 - */ -typedef int32_t UChar32; - -/** - * This value is intended for sentinel values for APIs that - * (take or) return single code points (UChar32). - * It is outside of the Unicode code point range 0..0x10ffff. - * - * For example, a "done" or "error" value in a new API - * could be indicated with U_SENTINEL. - * - * ICU APIs designed before ICU 2.4 usually define service-specific "done" - * values, mostly 0xffff. - * Those may need to be distinguished from - * actual U+ffff text contents by calling functions like - * CharacterIterator::hasNext() or UnicodeString::length(). - * - * @return -1 - * @see UChar32 - * @stable ICU 2.4 - */ -#define U_SENTINEL (-1) - -#include "./urename.h" - -#endif diff --git a/src/tree_sitter/unicode/urename.h b/src/tree_sitter/unicode/urename.h deleted file mode 100644 index ac79ad0f98..0000000000 --- a/src/tree_sitter/unicode/urename.h +++ /dev/null @@ -1 +0,0 @@ -// This file must exist in order for `utf8.h` and `utf16.h` to be used. diff --git a/src/tree_sitter/unicode/utf.h b/src/tree_sitter/unicode/utf.h deleted file mode 100644 index ac79ad0f98..0000000000 --- a/src/tree_sitter/unicode/utf.h +++ /dev/null @@ -1 +0,0 @@ -// This file must exist in order for `utf8.h` and `utf16.h` to be used. diff --git a/src/tree_sitter/unicode/utf16.h b/src/tree_sitter/unicode/utf16.h deleted file mode 100644 index b547922441..0000000000 --- a/src/tree_sitter/unicode/utf16.h +++ /dev/null @@ -1,733 +0,0 @@ -// © 2016 and later: Unicode, Inc. and others. -// License & terms of use: http://www.unicode.org/copyright.html -/* -******************************************************************************* -* -* Copyright (C) 1999-2012, International Business Machines -* Corporation and others. All Rights Reserved. -* -******************************************************************************* -* file name: utf16.h -* encoding: UTF-8 -* tab size: 8 (not used) -* indentation:4 -* -* created on: 1999sep09 -* created by: Markus W. Scherer -*/ - -/** - * \file - * \brief C API: 16-bit Unicode handling macros - * - * This file defines macros to deal with 16-bit Unicode (UTF-16) code units and strings. - * - * For more information see utf.h and the ICU User Guide Strings chapter - * (http://userguide.icu-project.org/strings). - * - * <em>Usage:</em> - * ICU coding guidelines for if() statements should be followed when using these macros. - * Compound statements (curly braces {}) must be used for if-else-while... - * bodies and all macro statements should be terminated with semicolon. - */ - -#ifndef __UTF16_H__ -#define __UTF16_H__ - -#include "./umachine.h" -#ifndef __UTF_H__ -# include "./utf.h" -#endif - -/* single-code point definitions -------------------------------------------- */ - -/** - * Does this code unit alone encode a code point (BMP, not a surrogate)? - * @param c 16-bit code unit - * @return TRUE or FALSE - * @stable ICU 2.4 - */ -#define U16_IS_SINGLE(c) !U_IS_SURROGATE(c) - -/** - * Is this code unit a lead surrogate (U+d800..U+dbff)? - * @param c 16-bit code unit - * @return TRUE or FALSE - * @stable ICU 2.4 - */ -#define U16_IS_LEAD(c) (((c)&0xfffffc00)==0xd800) - -/** - * Is this code unit a trail surrogate (U+dc00..U+dfff)? - * @param c 16-bit code unit - * @return TRUE or FALSE - * @stable ICU 2.4 - */ -#define U16_IS_TRAIL(c) (((c)&0xfffffc00)==0xdc00) - -/** - * Is this code unit a surrogate (U+d800..U+dfff)? - * @param c 16-bit code unit - * @return TRUE or FALSE - * @stable ICU 2.4 - */ -#define U16_IS_SURROGATE(c) U_IS_SURROGATE(c) - -/** - * Assuming c is a surrogate code point (U16_IS_SURROGATE(c)), - * is it a lead surrogate? - * @param c 16-bit code unit - * @return TRUE or FALSE - * @stable ICU 2.4 - */ -#define U16_IS_SURROGATE_LEAD(c) (((c)&0x400)==0) - -/** - * Assuming c is a surrogate code point (U16_IS_SURROGATE(c)), - * is it a trail surrogate? - * @param c 16-bit code unit - * @return TRUE or FALSE - * @stable ICU 4.2 - */ -#define U16_IS_SURROGATE_TRAIL(c) (((c)&0x400)!=0) - -/** - * Helper constant for U16_GET_SUPPLEMENTARY. - * @internal - */ -#define U16_SURROGATE_OFFSET ((0xd800<<10UL)+0xdc00-0x10000) - -/** - * Get a supplementary code point value (U+10000..U+10ffff) - * from its lead and trail surrogates. - * The result is undefined if the input values are not - * lead and trail surrogates. - * - * @param lead lead surrogate (U+d800..U+dbff) - * @param trail trail surrogate (U+dc00..U+dfff) - * @return supplementary code point (U+10000..U+10ffff) - * @stable ICU 2.4 - */ -#define U16_GET_SUPPLEMENTARY(lead, trail) \ - (((UChar32)(lead)<<10UL)+(UChar32)(trail)-U16_SURROGATE_OFFSET) - - -/** - * Get the lead surrogate (0xd800..0xdbff) for a - * supplementary code point (0x10000..0x10ffff). - * @param supplementary 32-bit code point (U+10000..U+10ffff) - * @return lead surrogate (U+d800..U+dbff) for supplementary - * @stable ICU 2.4 - */ -#define U16_LEAD(supplementary) (UChar)(((supplementary)>>10)+0xd7c0) - -/** - * Get the trail surrogate (0xdc00..0xdfff) for a - * supplementary code point (0x10000..0x10ffff). - * @param supplementary 32-bit code point (U+10000..U+10ffff) - * @return trail surrogate (U+dc00..U+dfff) for supplementary - * @stable ICU 2.4 - */ -#define U16_TRAIL(supplementary) (UChar)(((supplementary)&0x3ff)|0xdc00) - -/** - * How many 16-bit code units are used to encode this Unicode code point? (1 or 2) - * The result is not defined if c is not a Unicode code point (U+0000..U+10ffff). - * @param c 32-bit code point - * @return 1 or 2 - * @stable ICU 2.4 - */ -#define U16_LENGTH(c) ((uint32_t)(c)<=0xffff ? 1 : 2) - -/** - * The maximum number of 16-bit code units per Unicode code point (U+0000..U+10ffff). - * @return 2 - * @stable ICU 2.4 - */ -#define U16_MAX_LENGTH 2 - -/** - * Get a code point from a string at a random-access offset, - * without changing the offset. - * "Unsafe" macro, assumes well-formed UTF-16. - * - * The offset may point to either the lead or trail surrogate unit - * for a supplementary code point, in which case the macro will read - * the adjacent matching surrogate as well. - * The result is undefined if the offset points to a single, unpaired surrogate. - * Iteration through a string is more efficient with U16_NEXT_UNSAFE or U16_NEXT. - * - * @param s const UChar * string - * @param i string offset - * @param c output UChar32 variable - * @see U16_GET - * @stable ICU 2.4 - */ -#define U16_GET_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \ - (c)=(s)[i]; \ - if(U16_IS_SURROGATE(c)) { \ - if(U16_IS_SURROGATE_LEAD(c)) { \ - (c)=U16_GET_SUPPLEMENTARY((c), (s)[(i)+1]); \ - } else { \ - (c)=U16_GET_SUPPLEMENTARY((s)[(i)-1], (c)); \ - } \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Get a code point from a string at a random-access offset, - * without changing the offset. - * "Safe" macro, handles unpaired surrogates and checks for string boundaries. - * - * The offset may point to either the lead or trail surrogate unit - * for a supplementary code point, in which case the macro will read - * the adjacent matching surrogate as well. - * - * The length can be negative for a NUL-terminated string. - * - * If the offset points to a single, unpaired surrogate, then - * c is set to that unpaired surrogate. - * Iteration through a string is more efficient with U16_NEXT_UNSAFE or U16_NEXT. - * - * @param s const UChar * string - * @param start starting string offset (usually 0) - * @param i string offset, must be start<=i<length - * @param length string length - * @param c output UChar32 variable - * @see U16_GET_UNSAFE - * @stable ICU 2.4 - */ -#define U16_GET(s, start, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \ - (c)=(s)[i]; \ - if(U16_IS_SURROGATE(c)) { \ - uint16_t __c2; \ - if(U16_IS_SURROGATE_LEAD(c)) { \ - if((i)+1!=(length) && U16_IS_TRAIL(__c2=(s)[(i)+1])) { \ - (c)=U16_GET_SUPPLEMENTARY((c), __c2); \ - } \ - } else { \ - if((i)>(start) && U16_IS_LEAD(__c2=(s)[(i)-1])) { \ - (c)=U16_GET_SUPPLEMENTARY(__c2, (c)); \ - } \ - } \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Get a code point from a string at a random-access offset, - * without changing the offset. - * "Safe" macro, handles unpaired surrogates and checks for string boundaries. - * - * The offset may point to either the lead or trail surrogate unit - * for a supplementary code point, in which case the macro will read - * the adjacent matching surrogate as well. - * - * The length can be negative for a NUL-terminated string. - * - * If the offset points to a single, unpaired surrogate, then - * c is set to U+FFFD. - * Iteration through a string is more efficient with U16_NEXT_UNSAFE or U16_NEXT_OR_FFFD. - * - * @param s const UChar * string - * @param start starting string offset (usually 0) - * @param i string offset, must be start<=i<length - * @param length string length - * @param c output UChar32 variable - * @see U16_GET_UNSAFE - * @stable ICU 60 - */ -#define U16_GET_OR_FFFD(s, start, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \ - (c)=(s)[i]; \ - if(U16_IS_SURROGATE(c)) { \ - uint16_t __c2; \ - if(U16_IS_SURROGATE_LEAD(c)) { \ - if((i)+1!=(length) && U16_IS_TRAIL(__c2=(s)[(i)+1])) { \ - (c)=U16_GET_SUPPLEMENTARY((c), __c2); \ - } else { \ - (c)=0xfffd; \ - } \ - } else { \ - if((i)>(start) && U16_IS_LEAD(__c2=(s)[(i)-1])) { \ - (c)=U16_GET_SUPPLEMENTARY(__c2, (c)); \ - } else { \ - (c)=0xfffd; \ - } \ - } \ - } \ -} UPRV_BLOCK_MACRO_END - -/* definitions with forward iteration --------------------------------------- */ - -/** - * Get a code point from a string at a code point boundary offset, - * and advance the offset to the next code point boundary. - * (Post-incrementing forward iteration.) - * "Unsafe" macro, assumes well-formed UTF-16. - * - * The offset may point to the lead surrogate unit - * for a supplementary code point, in which case the macro will read - * the following trail surrogate as well. - * If the offset points to a trail surrogate, then that itself - * will be returned as the code point. - * The result is undefined if the offset points to a single, unpaired lead surrogate. - * - * @param s const UChar * string - * @param i string offset - * @param c output UChar32 variable - * @see U16_NEXT - * @stable ICU 2.4 - */ -#define U16_NEXT_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \ - (c)=(s)[(i)++]; \ - if(U16_IS_LEAD(c)) { \ - (c)=U16_GET_SUPPLEMENTARY((c), (s)[(i)++]); \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Get a code point from a string at a code point boundary offset, - * and advance the offset to the next code point boundary. - * (Post-incrementing forward iteration.) - * "Safe" macro, handles unpaired surrogates and checks for string boundaries. - * - * The length can be negative for a NUL-terminated string. - * - * The offset may point to the lead surrogate unit - * for a supplementary code point, in which case the macro will read - * the following trail surrogate as well. - * If the offset points to a trail surrogate or - * to a single, unpaired lead surrogate, then c is set to that unpaired surrogate. - * - * @param s const UChar * string - * @param i string offset, must be i<length - * @param length string length - * @param c output UChar32 variable - * @see U16_NEXT_UNSAFE - * @stable ICU 2.4 - */ -#define U16_NEXT(s, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \ - (c)=(s)[(i)++]; \ - if(U16_IS_LEAD(c)) { \ - uint16_t __c2; \ - if((i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \ - ++(i); \ - (c)=U16_GET_SUPPLEMENTARY((c), __c2); \ - } \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Get a code point from a string at a code point boundary offset, - * and advance the offset to the next code point boundary. - * (Post-incrementing forward iteration.) - * "Safe" macro, handles unpaired surrogates and checks for string boundaries. - * - * The length can be negative for a NUL-terminated string. - * - * The offset may point to the lead surrogate unit - * for a supplementary code point, in which case the macro will read - * the following trail surrogate as well. - * If the offset points to a trail surrogate or - * to a single, unpaired lead surrogate, then c is set to U+FFFD. - * - * @param s const UChar * string - * @param i string offset, must be i<length - * @param length string length - * @param c output UChar32 variable - * @see U16_NEXT_UNSAFE - * @stable ICU 60 - */ -#define U16_NEXT_OR_FFFD(s, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \ - (c)=(s)[(i)++]; \ - if(U16_IS_SURROGATE(c)) { \ - uint16_t __c2; \ - if(U16_IS_SURROGATE_LEAD(c) && (i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \ - ++(i); \ - (c)=U16_GET_SUPPLEMENTARY((c), __c2); \ - } else { \ - (c)=0xfffd; \ - } \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Append a code point to a string, overwriting 1 or 2 code units. - * The offset points to the current end of the string contents - * and is advanced (post-increment). - * "Unsafe" macro, assumes a valid code point and sufficient space in the string. - * Otherwise, the result is undefined. - * - * @param s const UChar * string buffer - * @param i string offset - * @param c code point to append - * @see U16_APPEND - * @stable ICU 2.4 - */ -#define U16_APPEND_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \ - if((uint32_t)(c)<=0xffff) { \ - (s)[(i)++]=(uint16_t)(c); \ - } else { \ - (s)[(i)++]=(uint16_t)(((c)>>10)+0xd7c0); \ - (s)[(i)++]=(uint16_t)(((c)&0x3ff)|0xdc00); \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Append a code point to a string, overwriting 1 or 2 code units. - * The offset points to the current end of the string contents - * and is advanced (post-increment). - * "Safe" macro, checks for a valid code point. - * If a surrogate pair is written, checks for sufficient space in the string. - * If the code point is not valid or a trail surrogate does not fit, - * then isError is set to TRUE. - * - * @param s const UChar * string buffer - * @param i string offset, must be i<capacity - * @param capacity size of the string buffer - * @param c code point to append - * @param isError output UBool set to TRUE if an error occurs, otherwise not modified - * @see U16_APPEND_UNSAFE - * @stable ICU 2.4 - */ -#define U16_APPEND(s, i, capacity, c, isError) UPRV_BLOCK_MACRO_BEGIN { \ - if((uint32_t)(c)<=0xffff) { \ - (s)[(i)++]=(uint16_t)(c); \ - } else if((uint32_t)(c)<=0x10ffff && (i)+1<(capacity)) { \ - (s)[(i)++]=(uint16_t)(((c)>>10)+0xd7c0); \ - (s)[(i)++]=(uint16_t)(((c)&0x3ff)|0xdc00); \ - } else /* c>0x10ffff or not enough space */ { \ - (isError)=TRUE; \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Advance the string offset from one code point boundary to the next. - * (Post-incrementing iteration.) - * "Unsafe" macro, assumes well-formed UTF-16. - * - * @param s const UChar * string - * @param i string offset - * @see U16_FWD_1 - * @stable ICU 2.4 - */ -#define U16_FWD_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ - if(U16_IS_LEAD((s)[(i)++])) { \ - ++(i); \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Advance the string offset from one code point boundary to the next. - * (Post-incrementing iteration.) - * "Safe" macro, handles unpaired surrogates and checks for string boundaries. - * - * The length can be negative for a NUL-terminated string. - * - * @param s const UChar * string - * @param i string offset, must be i<length - * @param length string length - * @see U16_FWD_1_UNSAFE - * @stable ICU 2.4 - */ -#define U16_FWD_1(s, i, length) UPRV_BLOCK_MACRO_BEGIN { \ - if(U16_IS_LEAD((s)[(i)++]) && (i)!=(length) && U16_IS_TRAIL((s)[i])) { \ - ++(i); \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Advance the string offset from one code point boundary to the n-th next one, - * i.e., move forward by n code points. - * (Post-incrementing iteration.) - * "Unsafe" macro, assumes well-formed UTF-16. - * - * @param s const UChar * string - * @param i string offset - * @param n number of code points to skip - * @see U16_FWD_N - * @stable ICU 2.4 - */ -#define U16_FWD_N_UNSAFE(s, i, n) UPRV_BLOCK_MACRO_BEGIN { \ - int32_t __N=(n); \ - while(__N>0) { \ - U16_FWD_1_UNSAFE(s, i); \ - --__N; \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Advance the string offset from one code point boundary to the n-th next one, - * i.e., move forward by n code points. - * (Post-incrementing iteration.) - * "Safe" macro, handles unpaired surrogates and checks for string boundaries. - * - * The length can be negative for a NUL-terminated string. - * - * @param s const UChar * string - * @param i int32_t string offset, must be i<length - * @param length int32_t string length - * @param n number of code points to skip - * @see U16_FWD_N_UNSAFE - * @stable ICU 2.4 - */ -#define U16_FWD_N(s, i, length, n) UPRV_BLOCK_MACRO_BEGIN { \ - int32_t __N=(n); \ - while(__N>0 && ((i)<(length) || ((length)<0 && (s)[i]!=0))) { \ - U16_FWD_1(s, i, length); \ - --__N; \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Adjust a random-access offset to a code point boundary - * at the start of a code point. - * If the offset points to the trail surrogate of a surrogate pair, - * then the offset is decremented. - * Otherwise, it is not modified. - * "Unsafe" macro, assumes well-formed UTF-16. - * - * @param s const UChar * string - * @param i string offset - * @see U16_SET_CP_START - * @stable ICU 2.4 - */ -#define U16_SET_CP_START_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ - if(U16_IS_TRAIL((s)[i])) { \ - --(i); \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Adjust a random-access offset to a code point boundary - * at the start of a code point. - * If the offset points to the trail surrogate of a surrogate pair, - * then the offset is decremented. - * Otherwise, it is not modified. - * "Safe" macro, handles unpaired surrogates and checks for string boundaries. - * - * @param s const UChar * string - * @param start starting string offset (usually 0) - * @param i string offset, must be start<=i - * @see U16_SET_CP_START_UNSAFE - * @stable ICU 2.4 - */ -#define U16_SET_CP_START(s, start, i) UPRV_BLOCK_MACRO_BEGIN { \ - if(U16_IS_TRAIL((s)[i]) && (i)>(start) && U16_IS_LEAD((s)[(i)-1])) { \ - --(i); \ - } \ -} UPRV_BLOCK_MACRO_END - -/* definitions with backward iteration -------------------------------------- */ - -/** - * Move the string offset from one code point boundary to the previous one - * and get the code point between them. - * (Pre-decrementing backward iteration.) - * "Unsafe" macro, assumes well-formed UTF-16. - * - * The input offset may be the same as the string length. - * If the offset is behind a trail surrogate unit - * for a supplementary code point, then the macro will read - * the preceding lead surrogate as well. - * If the offset is behind a lead surrogate, then that itself - * will be returned as the code point. - * The result is undefined if the offset is behind a single, unpaired trail surrogate. - * - * @param s const UChar * string - * @param i string offset - * @param c output UChar32 variable - * @see U16_PREV - * @stable ICU 2.4 - */ -#define U16_PREV_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \ - (c)=(s)[--(i)]; \ - if(U16_IS_TRAIL(c)) { \ - (c)=U16_GET_SUPPLEMENTARY((s)[--(i)], (c)); \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Move the string offset from one code point boundary to the previous one - * and get the code point between them. - * (Pre-decrementing backward iteration.) - * "Safe" macro, handles unpaired surrogates and checks for string boundaries. - * - * The input offset may be the same as the string length. - * If the offset is behind a trail surrogate unit - * for a supplementary code point, then the macro will read - * the preceding lead surrogate as well. - * If the offset is behind a lead surrogate or behind a single, unpaired - * trail surrogate, then c is set to that unpaired surrogate. - * - * @param s const UChar * string - * @param start starting string offset (usually 0) - * @param i string offset, must be start<i - * @param c output UChar32 variable - * @see U16_PREV_UNSAFE - * @stable ICU 2.4 - */ -#define U16_PREV(s, start, i, c) UPRV_BLOCK_MACRO_BEGIN { \ - (c)=(s)[--(i)]; \ - if(U16_IS_TRAIL(c)) { \ - uint16_t __c2; \ - if((i)>(start) && U16_IS_LEAD(__c2=(s)[(i)-1])) { \ - --(i); \ - (c)=U16_GET_SUPPLEMENTARY(__c2, (c)); \ - } \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Move the string offset from one code point boundary to the previous one - * and get the code point between them. - * (Pre-decrementing backward iteration.) - * "Safe" macro, handles unpaired surrogates and checks for string boundaries. - * - * The input offset may be the same as the string length. - * If the offset is behind a trail surrogate unit - * for a supplementary code point, then the macro will read - * the preceding lead surrogate as well. - * If the offset is behind a lead surrogate or behind a single, unpaired - * trail surrogate, then c is set to U+FFFD. - * - * @param s const UChar * string - * @param start starting string offset (usually 0) - * @param i string offset, must be start<i - * @param c output UChar32 variable - * @see U16_PREV_UNSAFE - * @stable ICU 60 - */ -#define U16_PREV_OR_FFFD(s, start, i, c) UPRV_BLOCK_MACRO_BEGIN { \ - (c)=(s)[--(i)]; \ - if(U16_IS_SURROGATE(c)) { \ - uint16_t __c2; \ - if(U16_IS_SURROGATE_TRAIL(c) && (i)>(start) && U16_IS_LEAD(__c2=(s)[(i)-1])) { \ - --(i); \ - (c)=U16_GET_SUPPLEMENTARY(__c2, (c)); \ - } else { \ - (c)=0xfffd; \ - } \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Move the string offset from one code point boundary to the previous one. - * (Pre-decrementing backward iteration.) - * The input offset may be the same as the string length. - * "Unsafe" macro, assumes well-formed UTF-16. - * - * @param s const UChar * string - * @param i string offset - * @see U16_BACK_1 - * @stable ICU 2.4 - */ -#define U16_BACK_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ - if(U16_IS_TRAIL((s)[--(i)])) { \ - --(i); \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Move the string offset from one code point boundary to the previous one. - * (Pre-decrementing backward iteration.) - * The input offset may be the same as the string length. - * "Safe" macro, handles unpaired surrogates and checks for string boundaries. - * - * @param s const UChar * string - * @param start starting string offset (usually 0) - * @param i string offset, must be start<i - * @see U16_BACK_1_UNSAFE - * @stable ICU 2.4 - */ -#define U16_BACK_1(s, start, i) UPRV_BLOCK_MACRO_BEGIN { \ - if(U16_IS_TRAIL((s)[--(i)]) && (i)>(start) && U16_IS_LEAD((s)[(i)-1])) { \ - --(i); \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Move the string offset from one code point boundary to the n-th one before it, - * i.e., move backward by n code points. - * (Pre-decrementing backward iteration.) - * The input offset may be the same as the string length. - * "Unsafe" macro, assumes well-formed UTF-16. - * - * @param s const UChar * string - * @param i string offset - * @param n number of code points to skip - * @see U16_BACK_N - * @stable ICU 2.4 - */ -#define U16_BACK_N_UNSAFE(s, i, n) UPRV_BLOCK_MACRO_BEGIN { \ - int32_t __N=(n); \ - while(__N>0) { \ - U16_BACK_1_UNSAFE(s, i); \ - --__N; \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Move the string offset from one code point boundary to the n-th one before it, - * i.e., move backward by n code points. - * (Pre-decrementing backward iteration.) - * The input offset may be the same as the string length. - * "Safe" macro, handles unpaired surrogates and checks for string boundaries. - * - * @param s const UChar * string - * @param start start of string - * @param i string offset, must be start<i - * @param n number of code points to skip - * @see U16_BACK_N_UNSAFE - * @stable ICU 2.4 - */ -#define U16_BACK_N(s, start, i, n) UPRV_BLOCK_MACRO_BEGIN { \ - int32_t __N=(n); \ - while(__N>0 && (i)>(start)) { \ - U16_BACK_1(s, start, i); \ - --__N; \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Adjust a random-access offset to a code point boundary after a code point. - * If the offset is behind the lead surrogate of a surrogate pair, - * then the offset is incremented. - * Otherwise, it is not modified. - * The input offset may be the same as the string length. - * "Unsafe" macro, assumes well-formed UTF-16. - * - * @param s const UChar * string - * @param i string offset - * @see U16_SET_CP_LIMIT - * @stable ICU 2.4 - */ -#define U16_SET_CP_LIMIT_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ - if(U16_IS_LEAD((s)[(i)-1])) { \ - ++(i); \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Adjust a random-access offset to a code point boundary after a code point. - * If the offset is behind the lead surrogate of a surrogate pair, - * then the offset is incremented. - * Otherwise, it is not modified. - * The input offset may be the same as the string length. - * "Safe" macro, handles unpaired surrogates and checks for string boundaries. - * - * The length can be negative for a NUL-terminated string. - * - * @param s const UChar * string - * @param start int32_t starting string offset (usually 0) - * @param i int32_t string offset, start<=i<=length - * @param length int32_t string length - * @see U16_SET_CP_LIMIT_UNSAFE - * @stable ICU 2.4 - */ -#define U16_SET_CP_LIMIT(s, start, i, length) UPRV_BLOCK_MACRO_BEGIN { \ - if((start)<(i) && ((i)<(length) || (length)<0) && U16_IS_LEAD((s)[(i)-1]) && U16_IS_TRAIL((s)[i])) { \ - ++(i); \ - } \ -} UPRV_BLOCK_MACRO_END - -#endif diff --git a/src/tree_sitter/unicode/utf8.h b/src/tree_sitter/unicode/utf8.h deleted file mode 100644 index 3b37873e37..0000000000 --- a/src/tree_sitter/unicode/utf8.h +++ /dev/null @@ -1,881 +0,0 @@ -// © 2016 and later: Unicode, Inc. and others. -// License & terms of use: http://www.unicode.org/copyright.html -/* -******************************************************************************* -* -* Copyright (C) 1999-2015, International Business Machines -* Corporation and others. All Rights Reserved. -* -******************************************************************************* -* file name: utf8.h -* encoding: UTF-8 -* tab size: 8 (not used) -* indentation:4 -* -* created on: 1999sep13 -* created by: Markus W. Scherer -*/ - -/** - * \file - * \brief C API: 8-bit Unicode handling macros - * - * This file defines macros to deal with 8-bit Unicode (UTF-8) code units (bytes) and strings. - * - * For more information see utf.h and the ICU User Guide Strings chapter - * (http://userguide.icu-project.org/strings). - * - * <em>Usage:</em> - * ICU coding guidelines for if() statements should be followed when using these macros. - * Compound statements (curly braces {}) must be used for if-else-while... - * bodies and all macro statements should be terminated with semicolon. - */ - -#ifndef __UTF8_H__ -#define __UTF8_H__ - -#include "./umachine.h" -#ifndef __UTF_H__ -# include "./utf.h" -#endif - -/* internal definitions ----------------------------------------------------- */ - -/** - * Counts the trail bytes for a UTF-8 lead byte. - * Returns 0 for 0..0xc1 as well as for 0xf5..0xff. - * leadByte might be evaluated multiple times. - * - * This is internal since it is not meant to be called directly by external clients; - * however it is called by public macros in this file and thus must remain stable. - * - * @param leadByte The first byte of a UTF-8 sequence. Must be 0..0xff. - * @internal - */ -#define U8_COUNT_TRAIL_BYTES(leadByte) \ - (U8_IS_LEAD(leadByte) ? \ - ((uint8_t)(leadByte)>=0xe0)+((uint8_t)(leadByte)>=0xf0)+1 : 0) - -/** - * Counts the trail bytes for a UTF-8 lead byte of a valid UTF-8 sequence. - * Returns 0 for 0..0xc1. Undefined for 0xf5..0xff. - * leadByte might be evaluated multiple times. - * - * This is internal since it is not meant to be called directly by external clients; - * however it is called by public macros in this file and thus must remain stable. - * - * @param leadByte The first byte of a UTF-8 sequence. Must be 0..0xff. - * @internal - */ -#define U8_COUNT_TRAIL_BYTES_UNSAFE(leadByte) \ - (((uint8_t)(leadByte)>=0xc2)+((uint8_t)(leadByte)>=0xe0)+((uint8_t)(leadByte)>=0xf0)) - -/** - * Mask a UTF-8 lead byte, leave only the lower bits that form part of the code point value. - * - * This is internal since it is not meant to be called directly by external clients; - * however it is called by public macros in this file and thus must remain stable. - * @internal - */ -#define U8_MASK_LEAD_BYTE(leadByte, countTrailBytes) ((leadByte)&=(1<<(6-(countTrailBytes)))-1) - -/** - * Internal bit vector for 3-byte UTF-8 validity check, for use in U8_IS_VALID_LEAD3_AND_T1. - * Each bit indicates whether one lead byte + first trail byte pair starts a valid sequence. - * Lead byte E0..EF bits 3..0 are used as byte index, - * first trail byte bits 7..5 are used as bit index into that byte. - * @see U8_IS_VALID_LEAD3_AND_T1 - * @internal - */ -#define U8_LEAD3_T1_BITS "\x20\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x10\x30\x30" - -/** - * Internal 3-byte UTF-8 validity check. - * Non-zero if lead byte E0..EF and first trail byte 00..FF start a valid sequence. - * @internal - */ -#define U8_IS_VALID_LEAD3_AND_T1(lead, t1) (U8_LEAD3_T1_BITS[(lead)&0xf]&(1<<((uint8_t)(t1)>>5))) - -/** - * Internal bit vector for 4-byte UTF-8 validity check, for use in U8_IS_VALID_LEAD4_AND_T1. - * Each bit indicates whether one lead byte + first trail byte pair starts a valid sequence. - * First trail byte bits 7..4 are used as byte index, - * lead byte F0..F4 bits 2..0 are used as bit index into that byte. - * @see U8_IS_VALID_LEAD4_AND_T1 - * @internal - */ -#define U8_LEAD4_T1_BITS "\x00\x00\x00\x00\x00\x00\x00\x00\x1E\x0F\x0F\x0F\x00\x00\x00\x00" - -/** - * Internal 4-byte UTF-8 validity check. - * Non-zero if lead byte F0..F4 and first trail byte 00..FF start a valid sequence. - * @internal - */ -#define U8_IS_VALID_LEAD4_AND_T1(lead, t1) (U8_LEAD4_T1_BITS[(uint8_t)(t1)>>4]&(1<<((lead)&7))) - -/** - * Function for handling "next code point" with error-checking. - * - * This is internal since it is not meant to be called directly by external clients; - * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this - * file and thus must remain stable, and should not be hidden when other internal - * functions are hidden (otherwise public macros would fail to compile). - * @internal - */ -U_STABLE UChar32 U_EXPORT2 -utf8_nextCharSafeBody(const uint8_t *s, int32_t *pi, int32_t length, UChar32 c, UBool strict); - -/** - * Function for handling "append code point" with error-checking. - * - * This is internal since it is not meant to be called directly by external clients; - * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this - * file and thus must remain stable, and should not be hidden when other internal - * functions are hidden (otherwise public macros would fail to compile). - * @internal - */ -U_STABLE int32_t U_EXPORT2 -utf8_appendCharSafeBody(uint8_t *s, int32_t i, int32_t length, UChar32 c, UBool *pIsError); - -/** - * Function for handling "previous code point" with error-checking. - * - * This is internal since it is not meant to be called directly by external clients; - * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this - * file and thus must remain stable, and should not be hidden when other internal - * functions are hidden (otherwise public macros would fail to compile). - * @internal - */ -U_STABLE UChar32 U_EXPORT2 -utf8_prevCharSafeBody(const uint8_t *s, int32_t start, int32_t *pi, UChar32 c, UBool strict); - -/** - * Function for handling "skip backward one code point" with error-checking. - * - * This is internal since it is not meant to be called directly by external clients; - * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this - * file and thus must remain stable, and should not be hidden when other internal - * functions are hidden (otherwise public macros would fail to compile). - * @internal - */ -U_STABLE int32_t U_EXPORT2 -utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i); - -/* single-code point definitions -------------------------------------------- */ - -/** - * Does this code unit (byte) encode a code point by itself (US-ASCII 0..0x7f)? - * @param c 8-bit code unit (byte) - * @return TRUE or FALSE - * @stable ICU 2.4 - */ -#define U8_IS_SINGLE(c) (((c)&0x80)==0) - -/** - * Is this code unit (byte) a UTF-8 lead byte? (0xC2..0xF4) - * @param c 8-bit code unit (byte) - * @return TRUE or FALSE - * @stable ICU 2.4 - */ -#define U8_IS_LEAD(c) ((uint8_t)((c)-0xc2)<=0x32) -// 0x32=0xf4-0xc2 - -/** - * Is this code unit (byte) a UTF-8 trail byte? (0x80..0xBF) - * @param c 8-bit code unit (byte) - * @return TRUE or FALSE - * @stable ICU 2.4 - */ -#define U8_IS_TRAIL(c) ((int8_t)(c)<-0x40) - -/** - * How many code units (bytes) are used for the UTF-8 encoding - * of this Unicode code point? - * @param c 32-bit code point - * @return 1..4, or 0 if c is a surrogate or not a Unicode code point - * @stable ICU 2.4 - */ -#define U8_LENGTH(c) \ - ((uint32_t)(c)<=0x7f ? 1 : \ - ((uint32_t)(c)<=0x7ff ? 2 : \ - ((uint32_t)(c)<=0xd7ff ? 3 : \ - ((uint32_t)(c)<=0xdfff || (uint32_t)(c)>0x10ffff ? 0 : \ - ((uint32_t)(c)<=0xffff ? 3 : 4)\ - ) \ - ) \ - ) \ - ) - -/** - * The maximum number of UTF-8 code units (bytes) per Unicode code point (U+0000..U+10ffff). - * @return 4 - * @stable ICU 2.4 - */ -#define U8_MAX_LENGTH 4 - -/** - * Get a code point from a string at a random-access offset, - * without changing the offset. - * The offset may point to either the lead byte or one of the trail bytes - * for a code point, in which case the macro will read all of the bytes - * for the code point. - * The result is undefined if the offset points to an illegal UTF-8 - * byte sequence. - * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT. - * - * @param s const uint8_t * string - * @param i string offset - * @param c output UChar32 variable - * @see U8_GET - * @stable ICU 2.4 - */ -#define U8_GET_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \ - int32_t _u8_get_unsafe_index=(int32_t)(i); \ - U8_SET_CP_START_UNSAFE(s, _u8_get_unsafe_index); \ - U8_NEXT_UNSAFE(s, _u8_get_unsafe_index, c); \ -} UPRV_BLOCK_MACRO_END - -/** - * Get a code point from a string at a random-access offset, - * without changing the offset. - * The offset may point to either the lead byte or one of the trail bytes - * for a code point, in which case the macro will read all of the bytes - * for the code point. - * - * The length can be negative for a NUL-terminated string. - * - * If the offset points to an illegal UTF-8 byte sequence, then - * c is set to a negative value. - * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT. - * - * @param s const uint8_t * string - * @param start int32_t starting string offset - * @param i int32_t string offset, must be start<=i<length - * @param length int32_t string length - * @param c output UChar32 variable, set to <0 in case of an error - * @see U8_GET_UNSAFE - * @stable ICU 2.4 - */ -#define U8_GET(s, start, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \ - int32_t _u8_get_index=(i); \ - U8_SET_CP_START(s, start, _u8_get_index); \ - U8_NEXT(s, _u8_get_index, length, c); \ -} UPRV_BLOCK_MACRO_END - -/** - * Get a code point from a string at a random-access offset, - * without changing the offset. - * The offset may point to either the lead byte or one of the trail bytes - * for a code point, in which case the macro will read all of the bytes - * for the code point. - * - * The length can be negative for a NUL-terminated string. - * - * If the offset points to an illegal UTF-8 byte sequence, then - * c is set to U+FFFD. - * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT_OR_FFFD. - * - * This macro does not distinguish between a real U+FFFD in the text - * and U+FFFD returned for an ill-formed sequence. - * Use U8_GET() if that distinction is important. - * - * @param s const uint8_t * string - * @param start int32_t starting string offset - * @param i int32_t string offset, must be start<=i<length - * @param length int32_t string length - * @param c output UChar32 variable, set to U+FFFD in case of an error - * @see U8_GET - * @stable ICU 51 - */ -#define U8_GET_OR_FFFD(s, start, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \ - int32_t _u8_get_index=(i); \ - U8_SET_CP_START(s, start, _u8_get_index); \ - U8_NEXT_OR_FFFD(s, _u8_get_index, length, c); \ -} UPRV_BLOCK_MACRO_END - -/* definitions with forward iteration --------------------------------------- */ - -/** - * Get a code point from a string at a code point boundary offset, - * and advance the offset to the next code point boundary. - * (Post-incrementing forward iteration.) - * "Unsafe" macro, assumes well-formed UTF-8. - * - * The offset may point to the lead byte of a multi-byte sequence, - * in which case the macro will read the whole sequence. - * The result is undefined if the offset points to a trail byte - * or an illegal UTF-8 sequence. - * - * @param s const uint8_t * string - * @param i string offset - * @param c output UChar32 variable - * @see U8_NEXT - * @stable ICU 2.4 - */ -#define U8_NEXT_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \ - (c)=(uint8_t)(s)[(i)++]; \ - if(!U8_IS_SINGLE(c)) { \ - if((c)<0xe0) { \ - (c)=(((c)&0x1f)<<6)|((s)[(i)++]&0x3f); \ - } else if((c)<0xf0) { \ - /* no need for (c&0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ \ - (c)=(UChar)(((c)<<12)|(((s)[i]&0x3f)<<6)|((s)[(i)+1]&0x3f)); \ - (i)+=2; \ - } else { \ - (c)=(((c)&7)<<18)|(((s)[i]&0x3f)<<12)|(((s)[(i)+1]&0x3f)<<6)|((s)[(i)+2]&0x3f); \ - (i)+=3; \ - } \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Get a code point from a string at a code point boundary offset, - * and advance the offset to the next code point boundary. - * (Post-incrementing forward iteration.) - * "Safe" macro, checks for illegal sequences and for string boundaries. - * - * The length can be negative for a NUL-terminated string. - * - * The offset may point to the lead byte of a multi-byte sequence, - * in which case the macro will read the whole sequence. - * If the offset points to a trail byte or an illegal UTF-8 sequence, then - * c is set to a negative value. - * - * @param s const uint8_t * string - * @param i int32_t string offset, must be i<length - * @param length int32_t string length - * @param c output UChar32 variable, set to <0 in case of an error - * @see U8_NEXT_UNSAFE - * @stable ICU 2.4 - */ -#define U8_NEXT(s, i, length, c) U8_INTERNAL_NEXT_OR_SUB(s, i, length, c, U_SENTINEL) - -/** - * Get a code point from a string at a code point boundary offset, - * and advance the offset to the next code point boundary. - * (Post-incrementing forward iteration.) - * "Safe" macro, checks for illegal sequences and for string boundaries. - * - * The length can be negative for a NUL-terminated string. - * - * The offset may point to the lead byte of a multi-byte sequence, - * in which case the macro will read the whole sequence. - * If the offset points to a trail byte or an illegal UTF-8 sequence, then - * c is set to U+FFFD. - * - * This macro does not distinguish between a real U+FFFD in the text - * and U+FFFD returned for an ill-formed sequence. - * Use U8_NEXT() if that distinction is important. - * - * @param s const uint8_t * string - * @param i int32_t string offset, must be i<length - * @param length int32_t string length - * @param c output UChar32 variable, set to U+FFFD in case of an error - * @see U8_NEXT - * @stable ICU 51 - */ -#define U8_NEXT_OR_FFFD(s, i, length, c) U8_INTERNAL_NEXT_OR_SUB(s, i, length, c, 0xfffd) - -/** @internal */ -#define U8_INTERNAL_NEXT_OR_SUB(s, i, length, c, sub) UPRV_BLOCK_MACRO_BEGIN { \ - (c)=(uint8_t)(s)[(i)++]; \ - if(!U8_IS_SINGLE(c)) { \ - uint8_t __t = 0; \ - if((i)!=(length) && \ - /* fetch/validate/assemble all but last trail byte */ \ - ((c)>=0xe0 ? \ - ((c)<0xf0 ? /* U+0800..U+FFFF except surrogates */ \ - U8_LEAD3_T1_BITS[(c)&=0xf]&(1<<((__t=(s)[i])>>5)) && \ - (__t&=0x3f, 1) \ - : /* U+10000..U+10FFFF */ \ - ((c)-=0xf0)<=4 && \ - U8_LEAD4_T1_BITS[(__t=(s)[i])>>4]&(1<<(c)) && \ - ((c)=((c)<<6)|(__t&0x3f), ++(i)!=(length)) && \ - (__t=(s)[i]-0x80)<=0x3f) && \ - /* valid second-to-last trail byte */ \ - ((c)=((c)<<6)|__t, ++(i)!=(length)) \ - : /* U+0080..U+07FF */ \ - (c)>=0xc2 && ((c)&=0x1f, 1)) && \ - /* last trail byte */ \ - (__t=(s)[i]-0x80)<=0x3f && \ - ((c)=((c)<<6)|__t, ++(i), 1)) { \ - } else { \ - (c)=(sub); /* ill-formed*/ \ - } \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Append a code point to a string, overwriting 1 to 4 bytes. - * The offset points to the current end of the string contents - * and is advanced (post-increment). - * "Unsafe" macro, assumes a valid code point and sufficient space in the string. - * Otherwise, the result is undefined. - * - * @param s const uint8_t * string buffer - * @param i string offset - * @param c code point to append - * @see U8_APPEND - * @stable ICU 2.4 - */ -#define U8_APPEND_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \ - uint32_t __uc=(c); \ - if(__uc<=0x7f) { \ - (s)[(i)++]=(uint8_t)__uc; \ - } else { \ - if(__uc<=0x7ff) { \ - (s)[(i)++]=(uint8_t)((__uc>>6)|0xc0); \ - } else { \ - if(__uc<=0xffff) { \ - (s)[(i)++]=(uint8_t)((__uc>>12)|0xe0); \ - } else { \ - (s)[(i)++]=(uint8_t)((__uc>>18)|0xf0); \ - (s)[(i)++]=(uint8_t)(((__uc>>12)&0x3f)|0x80); \ - } \ - (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \ - } \ - (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Append a code point to a string, overwriting 1 to 4 bytes. - * The offset points to the current end of the string contents - * and is advanced (post-increment). - * "Safe" macro, checks for a valid code point. - * If a non-ASCII code point is written, checks for sufficient space in the string. - * If the code point is not valid or trail bytes do not fit, - * then isError is set to TRUE. - * - * @param s const uint8_t * string buffer - * @param i int32_t string offset, must be i<capacity - * @param capacity int32_t size of the string buffer - * @param c UChar32 code point to append - * @param isError output UBool set to TRUE if an error occurs, otherwise not modified - * @see U8_APPEND_UNSAFE - * @stable ICU 2.4 - */ -#define U8_APPEND(s, i, capacity, c, isError) UPRV_BLOCK_MACRO_BEGIN { \ - uint32_t __uc=(c); \ - if(__uc<=0x7f) { \ - (s)[(i)++]=(uint8_t)__uc; \ - } else if(__uc<=0x7ff && (i)+1<(capacity)) { \ - (s)[(i)++]=(uint8_t)((__uc>>6)|0xc0); \ - (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \ - } else if((__uc<=0xd7ff || (0xe000<=__uc && __uc<=0xffff)) && (i)+2<(capacity)) { \ - (s)[(i)++]=(uint8_t)((__uc>>12)|0xe0); \ - (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \ - (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \ - } else if(0xffff<__uc && __uc<=0x10ffff && (i)+3<(capacity)) { \ - (s)[(i)++]=(uint8_t)((__uc>>18)|0xf0); \ - (s)[(i)++]=(uint8_t)(((__uc>>12)&0x3f)|0x80); \ - (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \ - (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \ - } else { \ - (isError)=TRUE; \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Advance the string offset from one code point boundary to the next. - * (Post-incrementing iteration.) - * "Unsafe" macro, assumes well-formed UTF-8. - * - * @param s const uint8_t * string - * @param i string offset - * @see U8_FWD_1 - * @stable ICU 2.4 - */ -#define U8_FWD_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ - (i)+=1+U8_COUNT_TRAIL_BYTES_UNSAFE((s)[i]); \ -} UPRV_BLOCK_MACRO_END - -/** - * Advance the string offset from one code point boundary to the next. - * (Post-incrementing iteration.) - * "Safe" macro, checks for illegal sequences and for string boundaries. - * - * The length can be negative for a NUL-terminated string. - * - * @param s const uint8_t * string - * @param i int32_t string offset, must be i<length - * @param length int32_t string length - * @see U8_FWD_1_UNSAFE - * @stable ICU 2.4 - */ -#define U8_FWD_1(s, i, length) UPRV_BLOCK_MACRO_BEGIN { \ - uint8_t __b=(s)[(i)++]; \ - if(U8_IS_LEAD(__b) && (i)!=(length)) { \ - uint8_t __t1=(s)[i]; \ - if((0xe0<=__b && __b<0xf0)) { \ - if(U8_IS_VALID_LEAD3_AND_T1(__b, __t1) && \ - ++(i)!=(length) && U8_IS_TRAIL((s)[i])) { \ - ++(i); \ - } \ - } else if(__b<0xe0) { \ - if(U8_IS_TRAIL(__t1)) { \ - ++(i); \ - } \ - } else /* c>=0xf0 */ { \ - if(U8_IS_VALID_LEAD4_AND_T1(__b, __t1) && \ - ++(i)!=(length) && U8_IS_TRAIL((s)[i]) && \ - ++(i)!=(length) && U8_IS_TRAIL((s)[i])) { \ - ++(i); \ - } \ - } \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Advance the string offset from one code point boundary to the n-th next one, - * i.e., move forward by n code points. - * (Post-incrementing iteration.) - * "Unsafe" macro, assumes well-formed UTF-8. - * - * @param s const uint8_t * string - * @param i string offset - * @param n number of code points to skip - * @see U8_FWD_N - * @stable ICU 2.4 - */ -#define U8_FWD_N_UNSAFE(s, i, n) UPRV_BLOCK_MACRO_BEGIN { \ - int32_t __N=(n); \ - while(__N>0) { \ - U8_FWD_1_UNSAFE(s, i); \ - --__N; \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Advance the string offset from one code point boundary to the n-th next one, - * i.e., move forward by n code points. - * (Post-incrementing iteration.) - * "Safe" macro, checks for illegal sequences and for string boundaries. - * - * The length can be negative for a NUL-terminated string. - * - * @param s const uint8_t * string - * @param i int32_t string offset, must be i<length - * @param length int32_t string length - * @param n number of code points to skip - * @see U8_FWD_N_UNSAFE - * @stable ICU 2.4 - */ -#define U8_FWD_N(s, i, length, n) UPRV_BLOCK_MACRO_BEGIN { \ - int32_t __N=(n); \ - while(__N>0 && ((i)<(length) || ((length)<0 && (s)[i]!=0))) { \ - U8_FWD_1(s, i, length); \ - --__N; \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Adjust a random-access offset to a code point boundary - * at the start of a code point. - * If the offset points to a UTF-8 trail byte, - * then the offset is moved backward to the corresponding lead byte. - * Otherwise, it is not modified. - * "Unsafe" macro, assumes well-formed UTF-8. - * - * @param s const uint8_t * string - * @param i string offset - * @see U8_SET_CP_START - * @stable ICU 2.4 - */ -#define U8_SET_CP_START_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ - while(U8_IS_TRAIL((s)[i])) { --(i); } \ -} UPRV_BLOCK_MACRO_END - -/** - * Adjust a random-access offset to a code point boundary - * at the start of a code point. - * If the offset points to a UTF-8 trail byte, - * then the offset is moved backward to the corresponding lead byte. - * Otherwise, it is not modified. - * - * "Safe" macro, checks for illegal sequences and for string boundaries. - * Unlike U8_TRUNCATE_IF_INCOMPLETE(), this macro always reads s[i]. - * - * @param s const uint8_t * string - * @param start int32_t starting string offset (usually 0) - * @param i int32_t string offset, must be start<=i - * @see U8_SET_CP_START_UNSAFE - * @see U8_TRUNCATE_IF_INCOMPLETE - * @stable ICU 2.4 - */ -#define U8_SET_CP_START(s, start, i) UPRV_BLOCK_MACRO_BEGIN { \ - if(U8_IS_TRAIL((s)[(i)])) { \ - (i)=utf8_back1SafeBody(s, start, (i)); \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * If the string ends with a UTF-8 byte sequence that is valid so far - * but incomplete, then reduce the length of the string to end before - * the lead byte of that incomplete sequence. - * For example, if the string ends with E1 80, the length is reduced by 2. - * - * In all other cases (the string ends with a complete sequence, or it is not - * possible for any further trail byte to extend the trailing sequence) - * the length remains unchanged. - * - * Useful for processing text split across multiple buffers - * (save the incomplete sequence for later) - * and for optimizing iteration - * (check for string length only once per character). - * - * "Safe" macro, checks for illegal sequences and for string boundaries. - * Unlike U8_SET_CP_START(), this macro never reads s[length]. - * - * (In UTF-16, simply check for U16_IS_LEAD(last code unit).) - * - * @param s const uint8_t * string - * @param start int32_t starting string offset (usually 0) - * @param length int32_t string length (usually start<=length) - * @see U8_SET_CP_START - * @stable ICU 61 - */ -#define U8_TRUNCATE_IF_INCOMPLETE(s, start, length) UPRV_BLOCK_MACRO_BEGIN { \ - if((length)>(start)) { \ - uint8_t __b1=s[(length)-1]; \ - if(U8_IS_SINGLE(__b1)) { \ - /* common ASCII character */ \ - } else if(U8_IS_LEAD(__b1)) { \ - --(length); \ - } else if(U8_IS_TRAIL(__b1) && ((length)-2)>=(start)) { \ - uint8_t __b2=s[(length)-2]; \ - if(0xe0<=__b2 && __b2<=0xf4) { \ - if(__b2<0xf0 ? U8_IS_VALID_LEAD3_AND_T1(__b2, __b1) : \ - U8_IS_VALID_LEAD4_AND_T1(__b2, __b1)) { \ - (length)-=2; \ - } \ - } else if(U8_IS_TRAIL(__b2) && ((length)-3)>=(start)) { \ - uint8_t __b3=s[(length)-3]; \ - if(0xf0<=__b3 && __b3<=0xf4 && U8_IS_VALID_LEAD4_AND_T1(__b3, __b2)) { \ - (length)-=3; \ - } \ - } \ - } \ - } \ -} UPRV_BLOCK_MACRO_END - -/* definitions with backward iteration -------------------------------------- */ - -/** - * Move the string offset from one code point boundary to the previous one - * and get the code point between them. - * (Pre-decrementing backward iteration.) - * "Unsafe" macro, assumes well-formed UTF-8. - * - * The input offset may be the same as the string length. - * If the offset is behind a multi-byte sequence, then the macro will read - * the whole sequence. - * If the offset is behind a lead byte, then that itself - * will be returned as the code point. - * The result is undefined if the offset is behind an illegal UTF-8 sequence. - * - * @param s const uint8_t * string - * @param i string offset - * @param c output UChar32 variable - * @see U8_PREV - * @stable ICU 2.4 - */ -#define U8_PREV_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \ - (c)=(uint8_t)(s)[--(i)]; \ - if(U8_IS_TRAIL(c)) { \ - uint8_t __b, __count=1, __shift=6; \ -\ - /* c is a trail byte */ \ - (c)&=0x3f; \ - for(;;) { \ - __b=(s)[--(i)]; \ - if(__b>=0xc0) { \ - U8_MASK_LEAD_BYTE(__b, __count); \ - (c)|=(UChar32)__b<<__shift; \ - break; \ - } else { \ - (c)|=(UChar32)(__b&0x3f)<<__shift; \ - ++__count; \ - __shift+=6; \ - } \ - } \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Move the string offset from one code point boundary to the previous one - * and get the code point between them. - * (Pre-decrementing backward iteration.) - * "Safe" macro, checks for illegal sequences and for string boundaries. - * - * The input offset may be the same as the string length. - * If the offset is behind a multi-byte sequence, then the macro will read - * the whole sequence. - * If the offset is behind a lead byte, then that itself - * will be returned as the code point. - * If the offset is behind an illegal UTF-8 sequence, then c is set to a negative value. - * - * @param s const uint8_t * string - * @param start int32_t starting string offset (usually 0) - * @param i int32_t string offset, must be start<i - * @param c output UChar32 variable, set to <0 in case of an error - * @see U8_PREV_UNSAFE - * @stable ICU 2.4 - */ -#define U8_PREV(s, start, i, c) UPRV_BLOCK_MACRO_BEGIN { \ - (c)=(uint8_t)(s)[--(i)]; \ - if(!U8_IS_SINGLE(c)) { \ - (c)=utf8_prevCharSafeBody((const uint8_t *)s, start, &(i), c, -1); \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Move the string offset from one code point boundary to the previous one - * and get the code point between them. - * (Pre-decrementing backward iteration.) - * "Safe" macro, checks for illegal sequences and for string boundaries. - * - * The input offset may be the same as the string length. - * If the offset is behind a multi-byte sequence, then the macro will read - * the whole sequence. - * If the offset is behind a lead byte, then that itself - * will be returned as the code point. - * If the offset is behind an illegal UTF-8 sequence, then c is set to U+FFFD. - * - * This macro does not distinguish between a real U+FFFD in the text - * and U+FFFD returned for an ill-formed sequence. - * Use U8_PREV() if that distinction is important. - * - * @param s const uint8_t * string - * @param start int32_t starting string offset (usually 0) - * @param i int32_t string offset, must be start<i - * @param c output UChar32 variable, set to U+FFFD in case of an error - * @see U8_PREV - * @stable ICU 51 - */ -#define U8_PREV_OR_FFFD(s, start, i, c) UPRV_BLOCK_MACRO_BEGIN { \ - (c)=(uint8_t)(s)[--(i)]; \ - if(!U8_IS_SINGLE(c)) { \ - (c)=utf8_prevCharSafeBody((const uint8_t *)s, start, &(i), c, -3); \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Move the string offset from one code point boundary to the previous one. - * (Pre-decrementing backward iteration.) - * The input offset may be the same as the string length. - * "Unsafe" macro, assumes well-formed UTF-8. - * - * @param s const uint8_t * string - * @param i string offset - * @see U8_BACK_1 - * @stable ICU 2.4 - */ -#define U8_BACK_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ - while(U8_IS_TRAIL((s)[--(i)])) {} \ -} UPRV_BLOCK_MACRO_END - -/** - * Move the string offset from one code point boundary to the previous one. - * (Pre-decrementing backward iteration.) - * The input offset may be the same as the string length. - * "Safe" macro, checks for illegal sequences and for string boundaries. - * - * @param s const uint8_t * string - * @param start int32_t starting string offset (usually 0) - * @param i int32_t string offset, must be start<i - * @see U8_BACK_1_UNSAFE - * @stable ICU 2.4 - */ -#define U8_BACK_1(s, start, i) UPRV_BLOCK_MACRO_BEGIN { \ - if(U8_IS_TRAIL((s)[--(i)])) { \ - (i)=utf8_back1SafeBody(s, start, (i)); \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Move the string offset from one code point boundary to the n-th one before it, - * i.e., move backward by n code points. - * (Pre-decrementing backward iteration.) - * The input offset may be the same as the string length. - * "Unsafe" macro, assumes well-formed UTF-8. - * - * @param s const uint8_t * string - * @param i string offset - * @param n number of code points to skip - * @see U8_BACK_N - * @stable ICU 2.4 - */ -#define U8_BACK_N_UNSAFE(s, i, n) UPRV_BLOCK_MACRO_BEGIN { \ - int32_t __N=(n); \ - while(__N>0) { \ - U8_BACK_1_UNSAFE(s, i); \ - --__N; \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Move the string offset from one code point boundary to the n-th one before it, - * i.e., move backward by n code points. - * (Pre-decrementing backward iteration.) - * The input offset may be the same as the string length. - * "Safe" macro, checks for illegal sequences and for string boundaries. - * - * @param s const uint8_t * string - * @param start int32_t index of the start of the string - * @param i int32_t string offset, must be start<i - * @param n number of code points to skip - * @see U8_BACK_N_UNSAFE - * @stable ICU 2.4 - */ -#define U8_BACK_N(s, start, i, n) UPRV_BLOCK_MACRO_BEGIN { \ - int32_t __N=(n); \ - while(__N>0 && (i)>(start)) { \ - U8_BACK_1(s, start, i); \ - --__N; \ - } \ -} UPRV_BLOCK_MACRO_END - -/** - * Adjust a random-access offset to a code point boundary after a code point. - * If the offset is behind a partial multi-byte sequence, - * then the offset is incremented to behind the whole sequence. - * Otherwise, it is not modified. - * The input offset may be the same as the string length. - * "Unsafe" macro, assumes well-formed UTF-8. - * - * @param s const uint8_t * string - * @param i string offset - * @see U8_SET_CP_LIMIT - * @stable ICU 2.4 - */ -#define U8_SET_CP_LIMIT_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ - U8_BACK_1_UNSAFE(s, i); \ - U8_FWD_1_UNSAFE(s, i); \ -} UPRV_BLOCK_MACRO_END - -/** - * Adjust a random-access offset to a code point boundary after a code point. - * If the offset is behind a partial multi-byte sequence, - * then the offset is incremented to behind the whole sequence. - * Otherwise, it is not modified. - * The input offset may be the same as the string length. - * "Safe" macro, checks for illegal sequences and for string boundaries. - * - * The length can be negative for a NUL-terminated string. - * - * @param s const uint8_t * string - * @param start int32_t starting string offset (usually 0) - * @param i int32_t string offset, must be start<=i<=length - * @param length int32_t string length - * @see U8_SET_CP_LIMIT_UNSAFE - * @stable ICU 2.4 - */ -#define U8_SET_CP_LIMIT(s, start, i, length) UPRV_BLOCK_MACRO_BEGIN { \ - if((start)<(i) && ((i)<(length) || (length)<0)) { \ - U8_BACK_1(s, start, i); \ - U8_FWD_1(s, i, length); \ - } \ -} UPRV_BLOCK_MACRO_END - -#endif |