update: complete windows test
This commit is contained in:
parent
7c452e14ac
commit
9c79196487
BIN
.vs/cmake.db
BIN
.vs/cmake.db
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
.vs/slnx.sqlite
BIN
.vs/slnx.sqlite
Binary file not shown.
|
|
@ -3,8 +3,8 @@
|
|||
javac -h cpp/include src/main/java/com/bonus/sdk/YoloSdk.java src/main/java/com/bonus/sdk/Detection.java
|
||||
|
||||
windows
|
||||
cmake -S cpp -B build/windows -G "Visual Studio 17 2022" -A x64 -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake
|
||||
|
||||
# (在 E:\sishu-yolo-sdk 目录下运行)
|
||||
cmake -S cpp -B build/windows -G "Visual Studio 17 2022" -A x64 -DCMAKE_TOOLCHAIN_FILE="E:/vcpkg/scripts/buildsystems/vcpkg.cmake"
|
||||
cmake --build build/windows --config Release
|
||||
|
||||
linux
|
||||
|
|
|
|||
|
|
@ -4,14 +4,18 @@ set(CMAKE_CXX_STANDARD 17)
|
|||
|
||||
|
||||
if(WIN32)
|
||||
find_package(JNI COMPONENTS JNI REQUIRED)
|
||||
set(JNI_INCLUDE_DIRS
|
||||
"C:/Program Files/Microsoft/jdk-11.0.29.7-hotspot/include"
|
||||
"C:/Program Files/Microsoft/jdk-11.0.29.7-hotspot/include/win32"
|
||||
)
|
||||
set(JNI_LIBRARIES "C:/Program Files/Microsoft/jdk-11.0.29.7-hotspot/lib/jvm.lib")
|
||||
message(STATUS "Configuring for Windows (using vcpkg OpenCV + manual ONNX)")
|
||||
|
||||
#
|
||||
find_package(OpenCV REQUIRED)
|
||||
|
||||
#
|
||||
set(ORT_MANUAL_PATH ${CMAKE_SOURCE_DIR}/../prebuilt_libs/onnxruntime-win-x64-1.23.2)
|
||||
set(ORT_MANUAL_PATH ${CMAKE_SOURCE_DIR}/../prebuilt_libs/onnxruntime-win-x64-1.18.0)
|
||||
|
||||
elseif(UNIX)
|
||||
set(JNI_INCLUDE_DIRS
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ YoloDetector::YoloDetector(const ORTCHAR_T* model_path, int in_width, int in_hei
|
|||
|
||||
Ort::SessionOptions session_options;
|
||||
|
||||
// session_options.AppendExecutionProvider_CPU(0);
|
||||
//session_options.AppendExecutionProvider("XnnpackExecutionProvider");
|
||||
|
||||
session = std::make_unique<Ort::Session>(env, model_path, session_options);
|
||||
|
||||
|
|
|
|||
3
pom.xml
3
pom.xml
|
|
@ -50,6 +50,9 @@
|
|||
<filtering>false</filtering>
|
||||
<includes>
|
||||
<include>my_yolo_sdk.dll</include>
|
||||
<include>abseil_dll.dll</include>
|
||||
<include>libprotobuf.dll</include>
|
||||
<include>zlib1.dll</include>
|
||||
<include>onnxruntime.dll</include>
|
||||
<include>opencv_core4.dll</include>
|
||||
<include>opencv_dnn4.dll</include>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
| FileName | Status | FileSize | TotalTime(sec) | Upload(sec) | Submit(sec) | SignWait(sec) | Retry Count |
|
||||
|---------------------------|--------|----------|----------------|-------------|-------------|---------------|-------------|
|
||||
| onnxruntime.dll | Pass | 10.3MB | 64.24 | 1.63 | 0.49 | 62.12 | 0 |
|
||||
| time_providers_shared.dll | Pass | 11.5KB | 74.9 | 0.97 | 0.43 | 72.78 | 0 |
|
||||
|
|
@ -0,0 +1 @@
|
|||
45737400a2f3015c11f005ed7603611eaed306a6
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
<p align="center"><img width="50%" src="docs/images/ONNX_Runtime_logo_dark.png" /></p>
|
||||
|
||||
**ONNX Runtime is a cross-platform inference and training machine-learning accelerator**.
|
||||
|
||||
**ONNX Runtime inference** can enable faster customer experiences and lower costs, supporting models from deep learning frameworks such as PyTorch and TensorFlow/Keras as well as classical machine learning libraries such as scikit-learn, LightGBM, XGBoost, etc. ONNX Runtime is compatible with different hardware, drivers, and operating systems, and provides optimal performance by leveraging hardware accelerators where applicable alongside graph optimizations and transforms. [Learn more →](https://www.onnxruntime.ai/docs/#onnx-runtime-for-inferencing)
|
||||
|
||||
**ONNX Runtime training** can accelerate the model training time on multi-node NVIDIA GPUs for transformer models with a one-line addition for existing PyTorch training scripts. [Learn more →](https://www.onnxruntime.ai/docs/#onnx-runtime-for-training)
|
||||
|
||||
## Get Started & Resources
|
||||
|
||||
* **General Information**: [onnxruntime.ai](https://onnxruntime.ai)
|
||||
|
||||
* **Usage documentation and tutorials**: [onnxruntime.ai/docs](https://onnxruntime.ai/docs)
|
||||
|
||||
* **YouTube video tutorials**: [youtube.com/@ONNXRuntime](https://www.youtube.com/@ONNXRuntime)
|
||||
|
||||
* [**Upcoming Release Roadmap**](https://github.com/microsoft/onnxruntime/wiki/Upcoming-Release-Roadmap)
|
||||
|
||||
* **Companion sample repositories**:
|
||||
- ONNX Runtime Inferencing: [microsoft/onnxruntime-inference-examples](https://github.com/microsoft/onnxruntime-inference-examples)
|
||||
- ONNX Runtime Training: [microsoft/onnxruntime-training-examples](https://github.com/microsoft/onnxruntime-training-examples)
|
||||
|
||||
## Builtin Pipeline Status
|
||||
|
||||
|System|Inference|Training|
|
||||
|---|---|---|
|
||||
|Windows|[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=9)<br>[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=10)<br>[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=47)||
|
||||
|Linux|[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=11)<br>[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=64)<br>[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=12)<br>[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=45)<br>[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=55)|[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=86)<br>[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=84)<br>[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=148)|
|
||||
|Mac|[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=13)||
|
||||
|Android|[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=53)||
|
||||
|iOS|[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=134)||
|
||||
|Web|[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=161)||
|
||||
|Other|[](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=187&repoName=microsoft%2Fonnxruntime)||
|
||||
|
||||
## Third-party Pipeline Status
|
||||
|
||||
|System|Inference|Training|
|
||||
|---|---|---|
|
||||
|Linux|[](https://github.com/Ascend/onnxruntime/actions/workflows/build-and-test.yaml)||
|
||||
|
||||
## Data/Telemetry
|
||||
|
||||
Windows distributions of this project may collect usage data and send it to Microsoft to help improve our products and services. See the [privacy statement](docs/Privacy.md) for more details.
|
||||
|
||||
## Contributions and Feedback
|
||||
|
||||
We welcome contributions! Please see the [contribution guidelines](CONTRIBUTING.md).
|
||||
|
||||
For feature requests or bug reports, please file a [GitHub Issue](https://github.com/Microsoft/onnxruntime/issues).
|
||||
|
||||
For general discussion or questions, please use [GitHub Discussions](https://github.com/microsoft/onnxruntime/discussions).
|
||||
|
||||
## Code of Conduct
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
||||
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
|
||||
or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the [MIT License](LICENSE).
|
||||
|
|
@ -2108,6 +2108,261 @@ SOFTWARE.
|
|||
|
||||
_____
|
||||
|
||||
TVM Open Deep Learning Compiler Stack
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright {yyyy} {name of copyright owner}
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
CONTRIBUTORS
|
||||
|
||||
TVM Contributors
|
||||
================
|
||||
TVM adopts the Apache style model and governs by merit. We believe that it is important to create an inclusive community where everyone can use,
|
||||
contribute to, and influence the direction of the project. We actively invite contributors who have earned the merit to be part of the development community.
|
||||
|
||||
See the [community structure document](http://docs.tvm.ai/contribute/community.html) for the explanation of community structure and contribution guidelines.
|
||||
|
||||
## Committers
|
||||
- [Tianqi Chen](https://github.com/tqchen) (PMC)
|
||||
- [Thierry Moreau](http://homes.cs.washington.edu/~moreau/)
|
||||
- [Ziheng Jiang](https://github.com/ZihengJiang)
|
||||
- [Haichen Shen](http://homes.cs.washington.edu/~haichen/)
|
||||
- [Yizhi Liu](https://github.com/yzhliu)
|
||||
|
||||
## Code Owners
|
||||
- [Aditya Atluri](https://github.com/adityaatluri) ROCM
|
||||
- [Leyuan Wang](https://github.com/Laurawly) TOPI
|
||||
- [Yuwei Hu](https://github.com/Huyuwei) TOPI
|
||||
- [Zhixun Tan](https://github.com/phisiart) OpenGL/WebGL backend
|
||||
- [Nick Hynes](https://github.com/nhynes) SGX and secured computing
|
||||
- [Lianmin Zheng](https://github.com/merrymercy) AutoTVM
|
||||
|
||||
## Reviewers
|
||||
- [Zhi Chen](https://github.com/zhiics)
|
||||
- [Xiaoqiang Dan](https://github.com/xqdan)
|
||||
- [Liangfu Chen](https://github.com/liangfu)
|
||||
- [Masahiro Masuda](https://github.com/masahi)
|
||||
- [Kazutaka Morita](https://github.com/kazum)
|
||||
- [Tatsuya Nishiyama](https://github.com/nishi-t)
|
||||
- [Pariksheet Pinjari](https://github.com/PariksheetPinjari909)
|
||||
- [Jared Roesch](https://github.com/jroesch)
|
||||
- [Siva](https://github.com/srkreddy1238)
|
||||
- [Siju Samuel](https://github.com/siju-samuel)
|
||||
- [Alex Weaver](https://github.com/alex-weaver)
|
||||
- [Yao Wang](https://github.com/kevinthesun)
|
||||
- [Jian Weng](https://github.com/were)
|
||||
- [Eddie Yan](https://github.com/eqy)
|
||||
- [Joshua Z. Zhang](https://github.com/zhreshold)
|
||||
|
||||
## List of Contributors
|
||||
- [Full List of Contributors](https://github.com/dmlc/tvm/graphs/contributors)
|
||||
- To contributors: please add your name to the list.
|
||||
- [Qiao Zhang](https://github.com/zhangqiaorjc)
|
||||
- [Haolong Zhang](https://github.com/haolongzhangm)
|
||||
- [Cody Hao Yu](https://github.com/comaniac)
|
||||
- [Chris Nuernberger](https://github.com/cnuernber)
|
||||
|
||||
_____
|
||||
|
||||
FreeBSD: getopt.c file
|
||||
|
||||
Copyright (c) 1987, 1993, 1994
|
||||
|
|
@ -2237,6 +2492,212 @@ DAMAGE.
|
|||
|
||||
_____
|
||||
|
||||
google/nsync
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
_____
|
||||
|
||||
google/re2
|
||||
|
||||
Copyright (c) 2009 The RE2 Authors. All rights reserved.
|
||||
|
|
@ -4359,7 +4820,7 @@ SOFTWARE.
|
|||
|
||||
----------------------------------------------------------------------------
|
||||
|
||||
This is the MIT/Expat License. For more information see:
|
||||
This is the MIT/Expat Licence. For more information see:
|
||||
|
||||
1. http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
|
|
@ -6045,112 +6506,3 @@ https://github.com/intel/neural-speed
|
|||
terms, and open source software license terms. These separate license terms
|
||||
govern your use of the third party programs as set forth in the
|
||||
"THIRD-PARTY-PROGRAMS" file.
|
||||
|
||||
_____
|
||||
|
||||
dawn
|
||||
|
||||
https://dawn.googlesource.com/dawn
|
||||
|
||||
BSD 3-Clause License
|
||||
|
||||
Copyright 2017-2023 The Dawn & Tint Authors
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
_____
|
||||
|
||||
KleidiAI
|
||||
|
||||
https://gitlab.arm.com/kleidi/kleidiai
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
Copyright 2024-2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
|
@ -0,0 +1 @@
|
|||
1.18.0
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -75,6 +75,8 @@ struct Float16Impl {
|
|||
static constexpr uint16_t kNegativeInfinityBits = 0xFC00U;
|
||||
static constexpr uint16_t kPositiveQNaNBits = 0x7E00U;
|
||||
static constexpr uint16_t kNegativeQNaNBits = 0xFE00U;
|
||||
static constexpr uint16_t kEpsilonBits = 0x4170U;
|
||||
static constexpr uint16_t kMinValueBits = 0xFBFFU; // Minimum normal number
|
||||
static constexpr uint16_t kMaxValueBits = 0x7BFFU; // Largest normal number
|
||||
static constexpr uint16_t kOneBits = 0x3C00U;
|
||||
static constexpr uint16_t kMinusOneBits = 0xBC00U;
|
||||
|
|
@ -362,6 +364,9 @@ struct BFloat16Impl {
|
|||
static constexpr uint16_t kNegativeInfinityBits = 0xFF80U;
|
||||
static constexpr uint16_t kPositiveQNaNBits = 0x7FC1U;
|
||||
static constexpr uint16_t kNegativeQNaNBits = 0xFFC1U;
|
||||
static constexpr uint16_t kSignaling_NaNBits = 0x7F80U;
|
||||
static constexpr uint16_t kEpsilonBits = 0x0080U;
|
||||
static constexpr uint16_t kMinValueBits = 0xFF7FU;
|
||||
static constexpr uint16_t kMaxValueBits = 0x7F7FU;
|
||||
static constexpr uint16_t kRoundToNearest = 0x7FFFU;
|
||||
static constexpr uint16_t kOneBits = 0x3F80U;
|
||||
|
|
@ -29,7 +29,7 @@ class ArgBase {
|
|||
ArgBase(OrtKernelContext* ctx,
|
||||
size_t indice,
|
||||
bool is_input) : ctx_(ctx), indice_(indice), is_input_(is_input) {}
|
||||
virtual ~ArgBase() {};
|
||||
virtual ~ArgBase(){};
|
||||
|
||||
protected:
|
||||
struct KernelContext ctx_;
|
||||
|
|
@ -361,7 +361,7 @@ struct TensorArray : public ArgBase {
|
|||
tensor = std::make_unique<Custom::Tensor<std::string>>(ctx, ith_input, true);
|
||||
break;
|
||||
default:
|
||||
ORT_CXX_API_THROW("unknown input type", ORT_RUNTIME_EXCEPTION);
|
||||
ORT_CXX_API_THROW("unknow input type", ORT_RUNTIME_EXCEPTION);
|
||||
break;
|
||||
}
|
||||
tensors_.emplace_back(tensor.release());
|
||||
|
|
@ -403,7 +403,7 @@ using Variadic = TensorArray;
|
|||
Note:
|
||||
OrtLiteCustomOp inherits from OrtCustomOp to bridge tween a custom func/struct and ort core.
|
||||
The lifetime of an OrtLiteCustomOp instance is managed by customer code, not ort, so:
|
||||
1. DO NOT cast OrtLiteCustomOp to OrtCustomOp and release since there is no virtual destructor in the hierarchy.
|
||||
1. DO NOT cast OrtLiteCustomOp to OrtCustomOp and release since there is no virtual destructor in the hierachy.
|
||||
2. OrtLiteCustomFunc and OrtLiteCustomStruct, as two sub-structs, can be released in form of OrtLiteCustomOp since all members are kept in the OrtLiteCustomOp,
|
||||
hence memory could still be recycled properly.
|
||||
Further, OrtCustomOp is a c struct bearing no v-table, so offspring structs are by design to be of zero virtual functions to maintain cast safety.
|
||||
|
|
@ -43,9 +43,6 @@ static const char* const kOrtRunOptionsConfigQnnPerfModePostRun = "qnn.htp_perf_
|
|||
// Set RPC control latency for QNN HTP backend
|
||||
static const char* const kOrtRunOptionsConfigQnnRpcControlLatency = "qnn.rpc_control_latency";
|
||||
|
||||
// Set QNN Lora Config File for apply Lora in QNN context binary
|
||||
static const char* const kOrtRunOptionsConfigQnnLoraConfig = "qnn.lora_config";
|
||||
|
||||
// Set graph annotation id for CUDA EP. Use with enable_cuda_graph=true.
|
||||
// The value should be an integer. If the value is not set, the default value is 0 and
|
||||
// ORT session only captures one cuda graph before another capture is requested.
|
||||
|
|
@ -10,10 +10,10 @@
|
|||
* "[Area][.[SubArea1].[SubArea2]...].[Keyname]"
|
||||
* Such as "ep.cuda.use_arena"
|
||||
* The Config Key cannot be empty
|
||||
* The maximum length of the Config Key is 1024
|
||||
* The maximum length of the Config Key is 128
|
||||
*
|
||||
* The string format of a SessionOptions Config Value is defined individually for each Config.
|
||||
* The maximum length of the Config Value is 2048
|
||||
* The maximum length of the Config Value is 1024
|
||||
*/
|
||||
|
||||
// Key for disable PrePacking,
|
||||
|
|
@ -67,10 +67,6 @@ static const char* const kOrtSessionOptionsEnableQuantQDQCleanup = "session.enab
|
|||
// GeluApproximation has side effects which may change the inference results. It is disabled by default due to this.
|
||||
static const char* const kOrtSessionOptionsEnableGeluApproximation = "optimization.enable_gelu_approximation";
|
||||
|
||||
// Enable or disable Cast chain elimination in graph optimization. "0": disable; "1": enable. The default is "0".
|
||||
// CastElimination with chain elimination has side effects which may change the inference results. It is disabled by default due to this.
|
||||
static const char* const kOrtSessionOptionsEnableCastChainElimination = "optimization.enable_cast_chain_elimination";
|
||||
|
||||
// This setting controls whether to enable AheadOfTime function inlining.
|
||||
// AOT function inlining examines the graph and attempts to inline as many locally defined functions in the model
|
||||
// as possible with the help of enabled execution providers.
|
||||
|
|
@ -82,20 +78,15 @@ static const char* const kOrtSessionOptionsEnableCastChainElimination = "optimiz
|
|||
static const char* const kOrtSessionOptionsDisableAheadOfTimeFunctionInlining = "session.disable_aot_function_inlining";
|
||||
|
||||
#ifdef ENABLE_TRAINING
|
||||
// Specifies a path of the file containing a list of memory optimization configurations.
|
||||
// The value should be a string indicating the file path of the config file.
|
||||
// The content of the config file is a JSON struct like this:
|
||||
// [
|
||||
// "Gelu+Cast+:1:0",
|
||||
// "Dropout+:1:1"
|
||||
// ]
|
||||
// Taking the example of "Gelu+Cast+:1:0",
|
||||
// > "Gelu+Cast+" is the subgraph string, a valid "subgraph string" should be one subgraph representation
|
||||
// output by ORT graph transformations.
|
||||
// > "1" is "optimization strategy", valid values: 0 - disabled, 1 - recompute.
|
||||
// > "0" is "number of subgraph to apply" which is used to control how many subgraphs to apply optimization,
|
||||
// to avoid "oversaving" the memory.
|
||||
static const char* const kOrtSessionOptionsMemoryOptimizerApplyConfig = "optimization.memory_optimizer_config";
|
||||
// Specifies a list of op types for memory footprint reduction.
|
||||
// The value should be a ","-delimited list of pair of
|
||||
// <subgraph string: optimization strategy: number of subgraph to apply>.
|
||||
// For example, "Gelu+Cast+:1:0,Dropout+:1:1".
|
||||
// A valid "subgraph string" should be one subgraph representation output by ORT graph transformations.
|
||||
// "optimization strategy" currently has valid values: 0 - disabled, 1 - recompute.
|
||||
// "number of subgraph to apply" is used to control how many subgraphs to apply optimization, to avoid "oversaving"
|
||||
// the memory.
|
||||
static const char* const kOrtSessionOptionsMemoryOptimizerEnabler = "optimization.memory_optimizer_config";
|
||||
|
||||
// Specifies the config for detecting subgraphs for memory footprint reduction.
|
||||
// The value should be a string contains int separated using commas. The default value is "0:0".
|
||||
|
|
@ -111,46 +102,13 @@ static const char* const kOrtSessionOptionsMemoryOptimizerProbeConfig = "optimiz
|
|||
// Default is an empty string which means no optimizers are disabled.
|
||||
static const char* const kOrtSessionOptionsDisableSpecifiedOptimizers = "optimization.disable_specified_optimizers";
|
||||
|
||||
// It controls whether to run graph optimizations in loop or not.
|
||||
//
|
||||
// "0": disable. Graph Optimization Loop is disabled.
|
||||
// ```
|
||||
// Level 2 --> Level 3 --> InsertCastTransforms --> Level 4
|
||||
// ^ |
|
||||
// | "No Loop" |
|
||||
// | |
|
||||
// X xxxxxxxxxxx X
|
||||
// ```
|
||||
// "1": enable. Graph Optimization Loop is enabled, such that, if optimizations at Level 4 are applied then
|
||||
// the loop will check for any other valid optimization that can happen.
|
||||
// ```
|
||||
// Level 2 --> Level 3 --> InsertCastTransforms --> Level 4
|
||||
// ^ |
|
||||
// | "Loop only depending on Level 4" |
|
||||
// | |
|
||||
// ---------------------------------------------------
|
||||
// ```
|
||||
// "2": enable. Graph Optimization Loop is enabled, such that, if optimizations at Level 2 or above are applied then
|
||||
// The loop will check for any other valid optimization that can happen.
|
||||
// ```
|
||||
// Level 2 --> Level 3 --> InsertCastTransforms --> Level 4
|
||||
// ^ |
|
||||
// | "Loop" |
|
||||
// | |
|
||||
// ---------------------------------------------------
|
||||
// ```
|
||||
// Default value is set to "1".
|
||||
static const char* const kOrtSessionOptionsGraphOptimizationsLoopLevel = "session.graph_optimizations_loop_level";
|
||||
|
||||
// Enable or disable using device allocator for allocating initialized tensor memory. "1": enable; "0": disable. The default is "0".
|
||||
// Using device allocators means the memory allocation is made using malloc/new.
|
||||
static const char* const kOrtSessionOptionsUseDeviceAllocatorForInitializers = "session.use_device_allocator_for_initializers";
|
||||
|
||||
// Configure whether to allow the inter_op/intra_op threads spinning a number of times before blocking
|
||||
// "0": thread will block if found no job to run
|
||||
// "1": thread will spin a number of times before blocking
|
||||
// The default is "0" when ORT is built with "ORT_CLIENT_PACKAGE_BUILD" and "1" otherwise.
|
||||
// Thread spinning is disabled by default for client/on-device workloads to reduce cpu utilization and improve power efficiency.
|
||||
// "1": default, thread will spin a number of times before blocking
|
||||
static const char* const kOrtSessionOptionsConfigAllowInterOpSpinning = "session.inter_op.allow_spinning";
|
||||
static const char* const kOrtSessionOptionsConfigAllowIntraOpSpinning = "session.intra_op.allow_spinning";
|
||||
|
||||
|
|
@ -287,51 +245,6 @@ static const char* const kOrtSessionOptionsOptimizedModelExternalInitializersFil
|
|||
static const char* const kOrtSessionOptionsOptimizedModelExternalInitializersMinSizeInBytes =
|
||||
"session.optimized_model_external_initializers_min_size_in_bytes";
|
||||
|
||||
// When loading model from memory buffer and the model has external initializers
|
||||
// Use this config to set the external data file folder path
|
||||
// All external data files should be in the same folder
|
||||
static const char* const kOrtSessionOptionsModelExternalInitializersFileFolderPath =
|
||||
"session.model_external_initializers_file_folder_path";
|
||||
|
||||
// Use this config when saving pre-packed constant initializers to an external data file.
|
||||
// This allows you to memory map pre-packed initializers on model load and leave it to
|
||||
// to the OS the amount of memory consumed by the pre-packed initializers. Otherwise,
|
||||
// pre-packed data resides on the heap.
|
||||
//
|
||||
// - "0": Default is not save pre-packed initializers to a data file.
|
||||
// - "1": Save pre-packed constant initializers to an external data file.
|
||||
// Sample usage: sess_options.add_session_config_entry(kOrtSessionOptionsSavePrePackedConstantInitializers, "1")
|
||||
static const char* const kOrtSessionOptionsSavePrePackedConstantInitializers =
|
||||
"session.save_external_prepacked_constant_initializers";
|
||||
|
||||
// Use this config when you want to collect memory stats for each node in the graph.
|
||||
// The file format is a CSV file with the following columns:
|
||||
// The file will be created if it does not exist, and will be overwritten if it does.
|
||||
//
|
||||
// The content of the file can be used to estimate memory requirements at run time including
|
||||
// the temporary allocations. This operation is preferably done on a CPU device, as the model may exceed
|
||||
// device memory limits in constrained environments. When enabling this option, it is important to disable
|
||||
// memory patterns, as they tend to allocate large blocks to avoid fragmentation and accommodate needs of multiple
|
||||
// kernels. Memory patterns may make it difficult to allocate on a device with limited memory.
|
||||
//
|
||||
// The collected stats then can be used to partition the graph among the devices in a way that only the
|
||||
// required memory is allocated on each device.
|
||||
//
|
||||
// node_name, initializers_memory, dynamic_outputs_sizes, temp_allocations_size
|
||||
//
|
||||
// - "full path to file": there is not a default for this option. If the file can not be opened for writing, an error will be returned.
|
||||
static const char* const kOrtSessionOptionsCollectNodeMemoryStatsToFile = "session.collect_node_memory_stats_to_file";
|
||||
|
||||
/// This is a composite CSV setting formatted as "memory limit in kb,file name for collected stats"
|
||||
/// "limit > 0": enables Capacity Aware Partitioning for Cuda EP. `limit` is optional and when absent
|
||||
/// the provider may attempt to figure out the memory available automatically.
|
||||
/// The setting with no limit is expected to look like: ",file name for collected stats"
|
||||
/// The EP will place nodes on device "file name" :
|
||||
/// this file is expected to be found at the same folder with the model. The file contains
|
||||
/// pre-recorded stats collected when running with kOrtSessionOptionsCollectNodeMemoryStatsToFile enforce (see above)
|
||||
static const char* const kOrtSessionOptionsResourceCudaPartitioningSettings =
|
||||
"session.resource_cuda_partitioning_settings";
|
||||
|
||||
// Enable EP context feature to dump the partitioned graph which includes the EP context into Onnx file.
|
||||
// The dumped Onnx model with EP context can be used for future inference to avoid the EP graph partitioning/compile overhead.
|
||||
// "0": disable. (default)
|
||||
|
|
@ -340,78 +253,15 @@ static const char* const kOrtSessionOptionEpContextEnable = "ep.context_enable";
|
|||
|
||||
// Specify the file path for the Onnx model which has EP context.
|
||||
// Default to original_file_name_ctx.onnx if not specified
|
||||
// Folder is not a valid option
|
||||
static const char* const kOrtSessionOptionEpContextFilePath = "ep.context_file_path";
|
||||
|
||||
// Flag to specify whether to dump the EP context into the Onnx model.
|
||||
// "0": dump the EP context into separate file, keep the file name in the Onnx model. (default).
|
||||
// "1": dump the EP context into the Onnx model.
|
||||
// "0": dump the EP context into separate file, keep the file name in the Onnx model.
|
||||
// "1": dump the EP context into the Onnx model. (default).
|
||||
static const char* const kOrtSessionOptionEpContextEmbedMode = "ep.context_embed_mode";
|
||||
|
||||
// Specify the EPContext node name prefix to make it unique
|
||||
// in case user need to merge/connect multiple EPContext nodes in one model
|
||||
static const char* const kOrtSessionOptionEpContextNodeNamePrefix = "ep.context_node_name_prefix";
|
||||
|
||||
// Share EP related resources across sessions
|
||||
static const char* const kOrtSessionOptionShareEpContexts = "ep.share_ep_contexts";
|
||||
|
||||
// Stop to share EP related resources across sessions from then on
|
||||
static const char* const kOrtSessionOptionStopShareEpContexts = "ep.stop_share_ep_contexts";
|
||||
|
||||
// Used only for context model generation.
|
||||
// This configuration is used when some nodes are partitioned on the CPU EP and those nodes have external initializers.
|
||||
// When generating the EP context model, the new model should not rely on the old external data file used by the source ONNX model.
|
||||
// Use this setting when dumping the EP context model with an external initializers file.
|
||||
// If specified, all initializers will be placed inside the external data file.
|
||||
// Otherwise, all initializers will be embedded inside the generated ONNX file.
|
||||
// By default, this option is not set, meaning all initializers will be included within the ONNX file.
|
||||
static const char* const kOrtSessionOptionsEpContextModelExternalInitializersFileName =
|
||||
"ep.context_model_external_initializers_file_name";
|
||||
|
||||
// Gemm fastmath mode provides fp32 gemm acceleration with bfloat16 based matmul.
|
||||
// Option values:
|
||||
// - "0": Gemm FastMath mode is not enabled. [DEFAULT]
|
||||
// - "1": Gemm FastMath mode is enabled.
|
||||
static const char* const kOrtSessionOptionsMlasGemmFastMathArm64Bfloat16 = "mlas.enable_gemm_fastmath_arm64_bfloat16";
|
||||
|
||||
// When converting DQ + MatMul -> MatMulNBits, the accuracy level of the MatMulNBits is controlled by this option.
|
||||
// Refer to MatMulNBits op schema for more details.
|
||||
// If not provided, default is 4.
|
||||
static const char* const kOrtSessionOptionsQDQMatMulNBitsAccuracyLevel = "session.qdq_matmulnbits_accuracy_level";
|
||||
|
||||
// THIS OPTION IS NOT A REGULAR SESSION OPTION SINCE IT CAN BE MODIFIED AT ANY TIME
|
||||
// Meant to be used with SetEpDynamicOptions
|
||||
// Specify the type of workload for this session.
|
||||
// "Default": OS determines the scheduling priority and processor performance to service this workload. [Default]
|
||||
// "Efficient": OS treats this workload is efficiency oriented with low scheduling priority and efficient processor performance.
|
||||
static const char* const kOrtEpDynamicOptionsWorkloadType = "ep.dynamic.workload_type";
|
||||
|
||||
// Disables model compilation during session initialization.
|
||||
//
|
||||
// If this option is set to "1", inference session creation will fail with error code ORT_MODEL_REQUIRES_COMPILATION
|
||||
// if compilation is required to run the model on any Execution Provider added to the session.
|
||||
// Only the following kinds of models are valid when this option is set to "1":
|
||||
// - Pre-compiled models that have EPContext nodes for the compiling Execution Providers in the session.
|
||||
// - Non-compiled models that run only on non-compiling Execution Providers, like CPU EP.
|
||||
//
|
||||
// See \href https://onnxruntime.ai/docs/execution-providers/EP-Context-Design.html for details about
|
||||
// compiled models with EPContext nodes.
|
||||
//
|
||||
// Option values:
|
||||
// - "0": EP compile is not disabled. [DEFAULT]
|
||||
// - "1": EP compile is disabled.
|
||||
static const char* const kOrtSessionOptionsDisableModelCompile = "session.disable_model_compile";
|
||||
|
||||
// Controls behavior when compiled model compatibility is SUPPORTED_PREFER_RECOMPILATION.
|
||||
// "0": Allow execution with suboptimal performance. [DEFAULT]
|
||||
// "1": Fail session creation to require recompilation for optimal performance.
|
||||
// Note: UNSUPPORTED models always fail regardless of this setting.
|
||||
static const char* const kOrtSessionOptionsFailOnSuboptimalCompiledModel =
|
||||
"session.fail_on_suboptimal_compiled_model";
|
||||
|
||||
// THIS OPTION IS NOT A REGULAR SESSION OPTION SINCE IT CAN BE MODIFIED AT ANY TIME
|
||||
// Meant to be used with SetEpDynamicOptions
|
||||
// options for HTP performance mode: "burst", "balanced", "default", "high_performance",
|
||||
// "high_power_saver", "low_balanced", "extreme_power_saver", "low_power_saver", "power_saver",
|
||||
// "sustained_high_performance". Default to "default".
|
||||
static const char* const kOrtEpDynamicOptionsQnnHtpPerformanceMode = "ep.dynamic.qnn_htp_performance_mode";
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
a83fc4d58cb48eb68890dd689f94f28288cf2278
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
<p align="center"><img width="50%" src="docs/images/ONNX_Runtime_logo_dark.png" /></p>
|
||||
|
||||
**ONNX Runtime is a cross-platform inference and training machine-learning accelerator**.
|
||||
|
||||
**ONNX Runtime inference** can enable faster customer experiences and lower costs, supporting models from deep learning frameworks such as PyTorch and TensorFlow/Keras as well as classical machine learning libraries such as scikit-learn, LightGBM, XGBoost, etc. ONNX Runtime is compatible with different hardware, drivers, and operating systems, and provides optimal performance by leveraging hardware accelerators where applicable alongside graph optimizations and transforms. [Learn more →](https://www.onnxruntime.ai/docs/#onnx-runtime-for-inferencing)
|
||||
|
||||
**ONNX Runtime training** can accelerate the model training time on multi-node NVIDIA GPUs for transformer models with a one-line addition for existing PyTorch training scripts. [Learn more →](https://www.onnxruntime.ai/docs/#onnx-runtime-for-training)
|
||||
|
||||
## Get Started & Resources
|
||||
|
||||
* **General Information**: [onnxruntime.ai](https://onnxruntime.ai)
|
||||
|
||||
* **Usage documentation and tutorials**: [onnxruntime.ai/docs](https://onnxruntime.ai/docs)
|
||||
|
||||
* **YouTube video tutorials**: [youtube.com/@ONNXRuntime](https://www.youtube.com/@ONNXRuntime)
|
||||
|
||||
* [**Upcoming Release Roadmap**](https://onnxruntime.ai/roadmap)
|
||||
|
||||
* **Companion sample repositories**:
|
||||
- ONNX Runtime Inferencing: [microsoft/onnxruntime-inference-examples](https://github.com/microsoft/onnxruntime-inference-examples)
|
||||
- ONNX Runtime Training: [microsoft/onnxruntime-training-examples](https://github.com/microsoft/onnxruntime-training-examples)
|
||||
|
||||
## Releases
|
||||
|
||||
The current release and past releases can be found here: https://github.com/microsoft/onnxruntime/releases.
|
||||
|
||||
For details on the upcoming release, including release dates, announcements, features, and guidance on submitting feature requests, please visit the release roadmap: https://onnxruntime.ai/roadmap.
|
||||
|
||||
## Data/Telemetry
|
||||
|
||||
Windows distributions of this project may collect usage data and send it to Microsoft to help improve our products and services. See the [privacy statement](docs/Privacy.md) for more details.
|
||||
|
||||
## Contributions and Feedback
|
||||
|
||||
We welcome contributions! Please see the [contribution guidelines](CONTRIBUTING.md).
|
||||
|
||||
For feature requests or bug reports, please file a [GitHub Issue](https://github.com/Microsoft/onnxruntime/issues).
|
||||
|
||||
For general discussion or questions, please use [GitHub Discussions](https://github.com/microsoft/onnxruntime/discussions).
|
||||
|
||||
## Code of Conduct
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
||||
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
|
||||
or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the [MIT License](LICENSE).
|
||||
|
|
@ -1 +0,0 @@
|
|||
1.23.2
|
||||
|
|
@ -1,988 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
// Do not include this file directly. Please include "onnxruntime_c_api.h" instead.
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
ORT_RUNTIME_CLASS(Ep);
|
||||
ORT_RUNTIME_CLASS(EpFactory);
|
||||
ORT_RUNTIME_CLASS(EpGraphSupportInfo);
|
||||
ORT_RUNTIME_CLASS(MemoryDevice); // opaque class to wrap onnxruntime::OrtDevice
|
||||
ORT_RUNTIME_CLASS(NodeComputeContext);
|
||||
|
||||
ORT_RUNTIME_CLASS(DataTransferImpl);
|
||||
ORT_RUNTIME_CLASS(SyncNotificationImpl);
|
||||
ORT_RUNTIME_CLASS(SyncStreamImpl);
|
||||
|
||||
// struct that an EP implements for IDataTransfer to copy between devices it uses and CPU
|
||||
struct OrtDataTransferImpl {
|
||||
uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION
|
||||
|
||||
/** \brief Release the OrtDataTransferImpl instance.
|
||||
*
|
||||
* This is called by ORT when the OrtDataTransferImpl instance is no longer needed.
|
||||
* The implementation should release any resources held by the instance.
|
||||
*
|
||||
* \param[in] this_ptr Pointer to the OrtDataTransferImpl instance.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(void, Release, _In_ OrtDataTransferImpl* this_ptr);
|
||||
|
||||
/** \brief Check if the implementation can copy between the source and destination memory devices.
|
||||
*
|
||||
* \param[in] this_ptr Pointer to the OrtDataTransferImpl instance.
|
||||
* \param[in] src_memory_device Source OrtMemoryDevice to copy from.
|
||||
* \param[in] dst_memory_device Destination OrtMemoryDevice to copy to.
|
||||
* \return True if the implementation can copy between the devices.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(bool, CanCopy, _In_ const OrtDataTransferImpl* this_ptr,
|
||||
_In_ const OrtMemoryDevice* src_memory_device, _In_ const OrtMemoryDevice* dst_memory_device);
|
||||
|
||||
/** \brief Copy tensors from src_tensors to dst_tensors using the provided streams.
|
||||
*
|
||||
* The implementation can use the provided streams to perform asynchronous copies if supported.
|
||||
* If a stream is not available, the copy is performed synchronously.
|
||||
*
|
||||
* \param[in] this_ptr Pointer to the OrtDataTransferImpl instance.
|
||||
* \param[in] src_tensors Array of source OrtValue pointers to copy from.
|
||||
* \param[in] dst_tensors Array of destination OrtValue pointers to copy to.
|
||||
* \param[in] streams Array of OrtSyncStream pointers for the copy operations, if the execution provider is stream
|
||||
* aware. nullptr if it is not.
|
||||
* \param[in] num_tensors Number of tensors to copy.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(CopyTensors, _In_ OrtDataTransferImpl* this_ptr,
|
||||
_In_reads_(num_tensors) const OrtValue** src_tensors,
|
||||
_In_reads_(num_tensors) OrtValue** dst_tensors,
|
||||
_In_reads_(num_tensors) OrtSyncStream** streams,
|
||||
_In_ size_t num_tensors);
|
||||
};
|
||||
|
||||
/** \brief Struct that an EP implements for Stream Notifications.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
struct OrtSyncNotificationImpl {
|
||||
uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION
|
||||
|
||||
/** \brief Release the OrtSyncNotificationImpl instance.
|
||||
*
|
||||
* This is called by ORT when the OrtSyncNotificationImpl instance is no longer needed.
|
||||
* The implementation should release any resources held by the instance.
|
||||
*
|
||||
* \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(void, Release, _In_ OrtSyncNotificationImpl* this_ptr);
|
||||
|
||||
/** \brief Called by ORT to activate the notification.
|
||||
*
|
||||
* \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(Activate, _In_ OrtSyncNotificationImpl* this_ptr);
|
||||
|
||||
/** \brief Wait for a device to device operation to complete.
|
||||
*
|
||||
* \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance.
|
||||
* \param[in] stream The OrtSyncStream instance that will wait on this notification to be activated.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(WaitOnDevice, _In_ OrtSyncNotificationImpl* this_ptr, _In_ OrtSyncStream* consumer_stream);
|
||||
|
||||
/** \brief Wait for a device to host operation to complete.
|
||||
*
|
||||
* \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(WaitOnHost, _In_ OrtSyncNotificationImpl* this_ptr);
|
||||
};
|
||||
|
||||
/** \brief Struct that an EP implements if it wishes to implement Stream support.
|
||||
*
|
||||
* This struct provides the overrides for onnxruntime::Stream's virtual methods.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
struct OrtSyncStreamImpl {
|
||||
uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION
|
||||
|
||||
/** \brief Release the OrtSyncStreamImpl instance.
|
||||
*
|
||||
* This is called by ORT when the OrtSyncStreamImpl instance is no longer needed.
|
||||
* The implementation should release any resources held by the instance.
|
||||
*
|
||||
* \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(void, Release, _In_ OrtSyncStreamImpl* this_ptr);
|
||||
|
||||
/** \brief Get the handle of the stream.
|
||||
*
|
||||
* This returns the native handle for the stream. e.g. cudaStream_t for CUDA streams.
|
||||
*
|
||||
* \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance.
|
||||
* \return The handle of the stream.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(void*, GetHandle, _In_ OrtSyncStreamImpl* this_ptr);
|
||||
|
||||
/** \brief Create an OrtSyncNotificationImpl for the OrtSyncStreamImpl instance.
|
||||
*
|
||||
* \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance
|
||||
* \param[out] notification The new OrtSyncNotificationImpl instance.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(CreateNotification, _In_ OrtSyncStreamImpl* this_ptr,
|
||||
_Outptr_ OrtSyncNotificationImpl** notification);
|
||||
|
||||
/** \brief Flush the stream.
|
||||
*
|
||||
* This is called by ORT to flush the stream, ensuring that all operations submitted to the stream are completed.
|
||||
*
|
||||
* \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(Flush, _In_ OrtSyncStreamImpl* this_ptr);
|
||||
|
||||
/** \brief Notify the stream that a session run has ended.
|
||||
*
|
||||
* This is called by ORT to notify the stream that a session run has ended, allowing the stream to perform any
|
||||
* necessary cleanup or finalization.
|
||||
*
|
||||
* \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(OnSessionRunEnd, _In_ OrtSyncStreamImpl* this_ptr);
|
||||
};
|
||||
|
||||
struct OrtNodeFusionOptions;
|
||||
typedef struct OrtNodeFusionOptions OrtNodeFusionOptions;
|
||||
|
||||
struct OrtNodeComputeInfo;
|
||||
typedef struct OrtNodeComputeInfo OrtNodeComputeInfo;
|
||||
|
||||
/**
|
||||
* \brief The OrtNodeFusionOptions struct specifies options for fusing nodes supported by an execution provider.
|
||||
*
|
||||
* Refer to OrtEpApi::EpGraphSupportInfo_AddNodesToFuse.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
struct OrtNodeFusionOptions {
|
||||
/** \brief The ONNX Runtime version the OrtNodeFusionOptions was compiled with.
|
||||
*
|
||||
* Implementation should set to ORT_API_VERSION.
|
||||
* ORT will use this to ensure it does not use members that were not available when the EP library was compiled.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
uint32_t ort_version_supported;
|
||||
|
||||
/** \brief If set to true, specify that the execution provider does not require ONNX Runtime to provide constant
|
||||
* initializers as inputs to the fused node during model inference. This is used when the execution
|
||||
* provider saves a copy of constant initializers, and allows ONNX Runtime to release constant initializers that
|
||||
* are not used by any execution provider.
|
||||
*
|
||||
* If not specified, defaults to false. That is, ONNX Runtime provides constant initializers as inputs to
|
||||
* the fused node by default.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
bool drop_constant_initializers;
|
||||
|
||||
// const OrtNode* fused_node_schema;
|
||||
};
|
||||
|
||||
/**
|
||||
* \brief The OrtNodeComputeInfo struct provides functions that an OrtEp implements to specify the compute
|
||||
* function for a compiled OrtGraph instance.
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
struct OrtNodeComputeInfo {
|
||||
/** \brief The ONNX Runtime version the OrtNodeComputeInfo was compiled with.
|
||||
*
|
||||
* Implementation should set to ORT_API_VERSION.
|
||||
* ORT will use this to ensure it does not call functions that were not available when the EP library was compiled.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
uint32_t ort_version_supported;
|
||||
|
||||
/** \brief Creates an opaque compute state object that is then passed to the Compute() function during inference.
|
||||
* \param[in] this_ptr The OrtNodeComputeInfo instance.
|
||||
* \param[in] compute_context OrtNodeComputeContext instance that contains compiled/fused node's name and host
|
||||
* memory allocation functions. Can optionally be used to build the compute state.
|
||||
* \param[out] compute_state Output parameter that is assigned the opaque computation state. ONNX Runtime calls
|
||||
* ReleaseState() (after calling Compute()) to allow the implementer to release the
|
||||
* compute state.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
OrtStatus*(ORT_API_CALL* CreateState)(_In_ OrtNodeComputeInfo* this_ptr,
|
||||
_In_ OrtNodeComputeContext* compute_context,
|
||||
_Outptr_ void** compute_state);
|
||||
|
||||
/** \brief Computation function called to execute the fused node compiled by an OrtEp instance.
|
||||
* \param[in] this_ptr The OrtNodeComputeInfo instance.
|
||||
* \param[in] compute_state The opaque computation state returned by CreateState().
|
||||
* \param[in] kernel_context The OrtKernelContext instance used to access inputs/outputs.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
OrtStatus*(ORT_API_CALL* Compute)(_In_ OrtNodeComputeInfo* this_ptr, _In_ void* compute_state,
|
||||
_In_ OrtKernelContext* kernel_context);
|
||||
|
||||
/** \brief Releases the compute state returned by CreateState().
|
||||
* \param[in] this_ptr The OrtNodeComputeInfo instance.
|
||||
* \param[inout] compute_state The opaque compute state returned by CreateState().
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
void(ORT_API_CALL* ReleaseState)(_In_ OrtNodeComputeInfo* this_ptr, _Frees_ptr_opt_ void* compute_state);
|
||||
};
|
||||
|
||||
struct OrtEpApi {
|
||||
/** \brief Create an OrtEpDevice for the EP and an OrtHardwareDevice.
|
||||
* \param[in] ep_factory Execution provider factory that is creating the instance.
|
||||
* \param[in] hardware_device Hardware device that the EP can utilize.
|
||||
* \param[in] ep_metadata Optional OrtKeyValuePairs instance for execution provider metadata that may be used
|
||||
* during execution provider selection and passed to CreateEp.
|
||||
* ep_device will copy this instance and the user should call ReleaseKeyValuePairs.
|
||||
* \param[in] ep_options Optional OrtKeyValuePairs instance for execution provider options that will be added
|
||||
* to the Session configuration options if the execution provider is selected.
|
||||
* ep_device will copy this instance and the user should call ReleaseKeyValuePairs.
|
||||
* \param ep_device OrtExecutionDevice that is created.
|
||||
*
|
||||
* \since Version 1.22.
|
||||
*/
|
||||
ORT_API2_STATUS(CreateEpDevice, _In_ OrtEpFactory* ep_factory,
|
||||
_In_ const OrtHardwareDevice* hardware_device,
|
||||
_In_opt_ const OrtKeyValuePairs* ep_metadata,
|
||||
_In_opt_ const OrtKeyValuePairs* ep_options,
|
||||
_Out_ OrtEpDevice** ep_device);
|
||||
|
||||
ORT_CLASS_RELEASE(EpDevice);
|
||||
|
||||
/** \brief Specify nodes that are supported by an OrtEp and should be fused into one node.
|
||||
*
|
||||
* Because the nodes will be fused into one "fused node", there must not exist an unsupported node in
|
||||
* a path between two of the provided nodes. Otherwise, the graph will become invalid.
|
||||
*
|
||||
* This function can be called multiple times. A subsequent call to this function will force the next set of
|
||||
* nodes to be fused into a different node.
|
||||
*
|
||||
* \param[in] graph_support_info OrtEpGraphSupportInfo instance to which to add the supported nodes.
|
||||
* \param[in] nodes Array of nodes supported by the EP that should be fused/compiled.
|
||||
* \param[in] num_nodes The number of supported nodes.
|
||||
* \param[in] node_fusion_options Optional node fusion options. Ignored if set to NULL.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(EpGraphSupportInfo_AddNodesToFuse, _In_ OrtEpGraphSupportInfo* graph_support_info,
|
||||
_In_reads_(num_nodes) const OrtNode* const* nodes, _In_ size_t num_nodes,
|
||||
_In_opt_ const OrtNodeFusionOptions* node_fusion_options);
|
||||
|
||||
/** \brief Specify a node that is supported by an OrtEp and should be run with a registered EP kernel.
|
||||
*
|
||||
* \param[in] graph_support_info OrtEpGraphSupportInfo instance to which to add the supported node.
|
||||
* \param[in] node The supported OrtNode instance.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(EpGraphSupportInfo_AddSingleNode, _In_ OrtEpGraphSupportInfo* graph_support_info,
|
||||
_In_ const OrtNode* node);
|
||||
|
||||
/** \brief Query a OrtNodeComputeContext for the name of the node that encapsulates the compiled/fused node.
|
||||
*
|
||||
* Used in OrtNodeComputeInfo::CreateComputeState().
|
||||
*
|
||||
* \param[in] context The OrtNodeComputeContext instance to query.
|
||||
* \return The node's name.
|
||||
*
|
||||
* \note Returned string is owned by ORT and valid only while OrtNodeComputeInfo::CreateComputeState() is called.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(const char*, NodeComputeContext_NodeName, _In_ const OrtNodeComputeContext* context);
|
||||
|
||||
/** \brief Register an allocator with the OrtEpDevice.
|
||||
*
|
||||
* This allows an EP to provide OrtMemoryInfo for DEFAULT and HOST_ACCESSIBLE memory type as needed.
|
||||
* The registered values will be used in calls to OrtEpFactory::CreateAllocator to ensure the required allocator/s
|
||||
* are available for EP usage.
|
||||
*
|
||||
* Multiple calls for the same entry type will replace a previous entry.
|
||||
*
|
||||
* Available entries:
|
||||
* - OrtDeviceAllocator with type of OrtDeviceMemoryType_DEFAULT
|
||||
* - OrtDeviceAllocator with type of OrtDeviceMemoryType_HOST_ACCESSIBLE
|
||||
* - OrtReadOnlyAllocator with type of OrtDeviceMemoryType_DEFAULT
|
||||
* - if provided this allocator will only be used to copy initializers to the device the EP uses.
|
||||
* ORT will use the OrtDeviceAllocator if not provided.
|
||||
*
|
||||
* \param[in] ep_device The OrtEpDevice instance to register the OrtMemoryInfo with.
|
||||
* \param[in] allocator_memory_info The OrtMemoryInfo information for the allocator.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(EpDevice_AddAllocatorInfo, _In_ OrtEpDevice* ep_device,
|
||||
_In_ const OrtMemoryInfo* allocator_memory_info);
|
||||
|
||||
/** \brief Get the OrtMemoryDevice from an OrtMemoryInfo instance.
|
||||
*
|
||||
* This is required for OrtDataTransferImpl (which implements onnxruntime::IDataTransfer) where the OrtMemoryDevice
|
||||
* is used in the CanCopy and CopyTensors functions.
|
||||
*
|
||||
* \param[in] memory_info The OrtMemoryInfo instance to get the memory device from.
|
||||
* \return The OrtMemoryDevice associated with the OrtMemoryInfo instance.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(const OrtMemoryDevice*, MemoryInfo_GetMemoryDevice, _In_ const OrtMemoryInfo* memory_info);
|
||||
|
||||
/** \brief Get the OrtMemoryDevice from an OrtValue instance if it contains a Tensor.
|
||||
*
|
||||
* \param[in] value The OrtValue instance to get the memory device from.
|
||||
* \return Memory device if OrtValue contains a Tensor, nullptr otherwise.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(const OrtMemoryDevice*, Value_GetMemoryDevice, _In_ const OrtValue* value);
|
||||
|
||||
/** \brief Compare two OrtMemoryDevice instances for equality.
|
||||
*
|
||||
* This is used to check if two memory devices are the same.
|
||||
* Used to implement DataTransferImpl::CanCopy.
|
||||
*
|
||||
* \param[in] a The first OrtMemoryDevice instance to compare.
|
||||
* \param[in] b The second OrtMemoryDevice instance to compare.
|
||||
* \return True if the two OrtMemoryDevice instances are equal, false otherwise.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(bool, MemoryDevice_AreEqual, _In_ const OrtMemoryDevice* a, _In_ const OrtMemoryDevice* b);
|
||||
|
||||
/** \brief Get the OrtMemoryInfoDeviceType value from an OrtMemoryDevice instance.
|
||||
*
|
||||
* \param[in] memory_device OrtMemoryDevice instance.
|
||||
* \return The OrtMemoryInfoDeviceType value.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(OrtMemoryInfoDeviceType, MemoryDevice_GetDeviceType, _In_ const OrtMemoryDevice* memory_device);
|
||||
|
||||
/** \brief Get the OrtDeviceMemoryType value from an OrtMemoryDevice instance.
|
||||
*
|
||||
* \param[in] memory_device OrtMemoryDevice instance.
|
||||
* \return The OrtDeviceMemoryType value.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(OrtDeviceMemoryType, MemoryDevice_GetMemoryType, _In_ const OrtMemoryDevice* memory_device);
|
||||
|
||||
/** \brief Get the vendor ID from an OrtMemoryDevice instance.
|
||||
*
|
||||
* The vendor ID is used to identify the vendor of the device, and is typically set to the PCI vendor ID.
|
||||
*
|
||||
* If the device is not vendor specific (e.g. CPU memory) the vendor ID is set to 0.
|
||||
*
|
||||
* \param[in] memory_device OrtMemoryDevice instance.
|
||||
* \return The vendor ID value.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(uint32_t, MemoryDevice_GetVendorId, _In_ const OrtMemoryDevice* memory_device);
|
||||
|
||||
/** \brief Get the device ID from an OrtMemoryDevice instance.
|
||||
*
|
||||
* \param[in] memory_device OrtMemoryDevice instance.
|
||||
* \return The device ID.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(uint32_t, MemoryDevice_GetDeviceId, _In_ const OrtMemoryDevice* memory_device);
|
||||
|
||||
/** \brief Get the OrtSyncStreamImpl associated with an OrtSyncStream instance.
|
||||
*
|
||||
* This allows an the plugin library to connect its OrtSyncStreamImpl instance with an OrtSyncStream if needed.
|
||||
*
|
||||
* \param[in] stream The OrtSyncStream instance to find an OrtSyncStreamImpl for.
|
||||
* \return The associated OrtSyncStreamImpl if found. nullptr otherwise.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*
|
||||
* \remarks There should always be an OrtSyncStreamImpl associated with an OrtSyncStream instance that the EP gets.
|
||||
*/
|
||||
ORT_API_T(const OrtSyncStreamImpl*, SyncStream_GetImpl, _In_ const OrtSyncStream* stream);
|
||||
|
||||
/** \brief Get the current sync ID for a stream.
|
||||
*
|
||||
* \param[in] stream The OrtSyncStream to get the sync ID for.
|
||||
* \return Current sync ID.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(uint64_t, SyncStream_GetSyncId, _In_ const OrtSyncStream* stream);
|
||||
|
||||
/** \brief Get the sync ID for the last time the consumer_stream waited on the producer_stream.
|
||||
*
|
||||
* When two streams are synchronized, the sync id represents the event used in that synchronization.
|
||||
*
|
||||
* \param[in] producer_stream The OrtSyncStream that produced the data.
|
||||
* \param[in] consumer_stream The OrtSyncStream that waited on the producer_stream.
|
||||
* \return ID for last sync. 0 if no sync has occurred between the two streams.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(uint64_t, GetSyncIdForLastWaitOnSyncStream,
|
||||
_In_ const OrtSyncStream* producer_stream, _In_ const OrtSyncStream* consumer_stream);
|
||||
};
|
||||
|
||||
/**
|
||||
* \brief The data layout type.
|
||||
*
|
||||
* EPs may specify a preferred data layout type. ORT's default layout type is OrtEpDataLayout_NCHW, or
|
||||
* OrtEpDataLayout_Default.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
typedef enum OrtEpDataLayout {
|
||||
OrtEpDataLayout_NCHW = 0,
|
||||
OrtEpDataLayout_NHWC,
|
||||
|
||||
OrtEpDataLayout_Default = OrtEpDataLayout_NCHW,
|
||||
} OrtEpDataLayout;
|
||||
|
||||
/**
|
||||
* \brief The OrtEp struct provides functions to implement for an execution provider.
|
||||
* \since Version 1.22.
|
||||
*/
|
||||
struct OrtEp {
|
||||
/** \brief The ONNX Runtime version the execution provider was compiled with.
|
||||
*
|
||||
* Implementation should set to ORT_API_VERSION.
|
||||
* ORT will use this to ensure it does not call functions that were not available when the library was compiled.
|
||||
*
|
||||
* \since Version 1.22.
|
||||
*/
|
||||
uint32_t ort_version_supported;
|
||||
|
||||
/** \brief Get the execution provider name.
|
||||
*
|
||||
* The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEp instance.
|
||||
* \return The execution provider name.
|
||||
*
|
||||
* \since Version 1.22.
|
||||
*/
|
||||
ORT_API_T(const char*, GetName, _In_ const OrtEp* this_ptr);
|
||||
|
||||
/** \brief Get information about the nodes supported by the OrtEp instance.
|
||||
*
|
||||
* IMPORTANT: This is not the final version of this API function. This is currently experimental but will
|
||||
* be stabilized by the ONNX Runtime 1.23 release.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEp instance.
|
||||
* \param[in] graph The OrtGraph instance for which to populate node support. The OrtGraph could be a nested subgraph
|
||||
* contained by a node (e.g., an If or Loop node). ONNX Runtime calls this function separately
|
||||
* for each nested subgraph.
|
||||
* \param[inout] graph_support_info OrtEpGraphSupportInfo instance that the implementer must fill out in order to
|
||||
* specify the supported nodes.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(GetCapability, _In_ OrtEp* this_ptr, _In_ const OrtGraph* graph,
|
||||
_Inout_ OrtEpGraphSupportInfo* graph_support_info);
|
||||
|
||||
/** \brief Compile OrtGraph instances assigned to the OrtEp. Implementer must set a OrtNodeComputeInfo instance
|
||||
* for each OrtGraph in order to define its computation function.
|
||||
*
|
||||
* If the session is configured to generate a pre-compiled model, the execution provider must return EPContext nodes,
|
||||
* as OrtNode instances, that ONNX Runtime uses to create a pre-compiled model, known as an "EPContext model".
|
||||
* An EPContext model contains EPContext nodes. Each EPContext node encapsulates the pre-compiled binary data for a
|
||||
* OrtGraph compiled for a specific execution provider. For more details about the EPContext design, refer to:
|
||||
* \htmlonly
|
||||
* <a href="https://onnxruntime.ai/docs/execution-providers/EP-Context-Design.html">EPContext design document.</a>
|
||||
* \endhtmlonly
|
||||
*
|
||||
* \param[in] this_ptr The OrtEp instance.
|
||||
* \param[in] graphs Array of `count` OrtGraph instances to compile. Each graph contains only the nodes for
|
||||
* which the execution provider indicated support. Nested subgraphs contained by a
|
||||
* node, such as an If or Loop, have separate OrtGraph instances.
|
||||
* \param[in] fused_nodes Array of `count` fused nodes that will replace the compiled graphs.
|
||||
* Each fused node is an OrtNode initialized with the intended fused node name and
|
||||
* input/output information.
|
||||
* \param[in] count The number of OrtGraph instances to compile.
|
||||
* \param[out] node_compute_infos Array of `count` OrtNodeComputeInfo instances that define each OrtGraph instance's
|
||||
* computation function. The implementer allocates the OrtNodeComputeInfo instances.
|
||||
* ORT calls ReleaseNodeComputeInfos() to release multiple instances in a batch.
|
||||
* \param[out] ep_context_nodes Output array of `count` OrtNode instances, each representing an EPContext
|
||||
* node for a compiled OrtGraph. The execution provider must use
|
||||
* OrtModelEditorApi::CreateNode to create the OrtNode instances. ONNX Runtime takes
|
||||
* ownership of the OrtNode instances, so the execution provider must NOT call
|
||||
* OrtApi::ReleaseNode. Should be ignored if the session is not configured to generate an
|
||||
* EPContext model.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \note Do NOT cache the provided OrtGraph instances in any of the OrtNodeComputeInfo functions because the
|
||||
* graphs are only valid for the duration of the call to Compile. Any graph/node/input/output
|
||||
* names that are needed by the OrtNodeComputeInfo functions must be copied and stored by the OrtEp.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(Compile, _In_ OrtEp* this_ptr, _In_ const OrtGraph** graphs,
|
||||
_In_ const OrtNode** fused_nodes, _In_ size_t count,
|
||||
_Out_writes_all_(count) OrtNodeComputeInfo** node_compute_infos,
|
||||
_Out_writes_(count) OrtNode** ep_context_nodes);
|
||||
|
||||
/** \brief Release OrtNodeComputeInfo instances.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEp instance.
|
||||
* \param[inout] node_compute_infos The OrtNodeComputeInfo instances to release.
|
||||
* \param[in] num_node_compute_infos The number of OrtNodeComputeInfo instances.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(void, ReleaseNodeComputeInfos, _In_ OrtEp* this_ptr,
|
||||
OrtNodeComputeInfo** node_compute_infos,
|
||||
_In_ size_t num_node_compute_infos);
|
||||
|
||||
/** \brief Get the EP's preferred data layout.
|
||||
*
|
||||
* \note Implementation of this function is optional.
|
||||
* If not implemented, ORT will assume that this EP prefers the data layout `OrtEpDataLayout::NCHW`.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEp instance.
|
||||
* \param[out] preferred_data_layout The EP's preferred data layout.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(GetPreferredDataLayout, _In_ OrtEp* this_ptr, _Out_ OrtEpDataLayout* preferred_data_layout);
|
||||
|
||||
/** \brief Given an op with domain `domain` and type `op_type`, determine whether an associated node's data layout
|
||||
* should be converted to `target_data_layout`.
|
||||
* If the EP prefers a non-default data layout (see `GetPreferredDataLayout()`), this function will be called
|
||||
* during layout transformation with `target_data_layout` set to the EP's preferred data layout.
|
||||
*
|
||||
* \note Implementation of this function is optional.
|
||||
* If an EP prefers a non-default data layout, it may implement this to customize the specific op data layout
|
||||
* preferences at a finer granularity.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEp instance.
|
||||
* \param[in] domain The op domain. An empty string means the ONNX domain.
|
||||
* \param[in] op_type The op type.
|
||||
* \param[in] target_data_layout The target data layout.
|
||||
* \param[out] should_convert Whether the associated node's data layout should be converted to `target_data_layout`.
|
||||
* If greater than 0, convert.
|
||||
* If 0, don't convert.
|
||||
* Otherwise, if less than 0, leave the decision to ORT.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(ShouldConvertDataLayoutForOp, _In_ OrtEp* this_ptr,
|
||||
_In_z_ const char* domain, _In_z_ const char* op_type,
|
||||
_In_ OrtEpDataLayout target_data_layout,
|
||||
_Outptr_ int* should_convert);
|
||||
|
||||
/** \brief Set dynamic options on this EP.
|
||||
*
|
||||
* Dynamic options can be set by the user at any time after session creation with `OrtApi::SetEpDynamicOptions()`.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEp instance.
|
||||
* \param[in] option_keys The dynamic option keys.
|
||||
* \param[in] option_values The dynamic option values.
|
||||
* \param[in] num_options The number of dynamic options.
|
||||
*
|
||||
* \note Implementation of this function is optional.
|
||||
* An EP should only implement this if it needs to handle any dynamic options.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(SetDynamicOptions, _In_ OrtEp* this_ptr,
|
||||
_In_reads_(num_options) const char* const* option_keys,
|
||||
_In_reads_(num_options) const char* const* option_values,
|
||||
_In_ size_t num_options);
|
||||
|
||||
/** \brief Called by ORT to notify the EP of the start of a run.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEp instance.
|
||||
* \param[in] run_options The run options for this run.
|
||||
*
|
||||
* \note Implementation of this function is optional.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(OnRunStart, _In_ OrtEp* this_ptr, _In_ const OrtRunOptions* run_options);
|
||||
|
||||
/** \brief Called by ORT to notify the EP of the end of a run.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEp instance.
|
||||
* \param[in] run_options The run options for this run.
|
||||
* \param[in] sync_stream Whether any associated stream should be synchronized during this call.
|
||||
* Only applicable if there is such a stream.
|
||||
*
|
||||
* \note Implementation of this function is optional.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(OnRunEnd, _In_ OrtEp* this_ptr, _In_ const OrtRunOptions* run_options, _In_ bool sync_stream);
|
||||
|
||||
/** \brief Create an OrtAllocator for the given OrtMemoryInfo for an OrtSession.
|
||||
*
|
||||
* The OrtMemoryInfo instance will match one of the values set in the OrtEpDevice using EpDevice_AddAllocatorInfo.
|
||||
* Any allocator specific options should be read from the session options.
|
||||
*
|
||||
* If nullptr OrtEpFactory::CreateAllocator will be used.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEpFactory instance.
|
||||
* \param[in] memory_info The OrtMemoryInfo to create the allocator for. May be nullptr.
|
||||
* \param[out] allocator The created OrtAllocator instance. Set to nullptr if the default CPU allocator is used.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(CreateAllocator, _In_ OrtEp* this_ptr,
|
||||
_In_ const OrtMemoryInfo* memory_info,
|
||||
_Outptr_result_maybenull_ OrtAllocator** allocator);
|
||||
|
||||
/** \brief Create a synchronization stream for the given memory device for an OrtSession.
|
||||
*
|
||||
* This is used to create a synchronization stream for the execution provider and is used to synchronize
|
||||
* operations on the device during model execution.
|
||||
* Any stream specific options should be read from the session options.
|
||||
*
|
||||
* If nullptr OrtEpFactory::CreateSyncStreamForDevice will be used.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEpFactory instance.
|
||||
* \param[in] memory_device The OrtMemoryDevice to create the synchronization stream for.
|
||||
* \param[out] stream The created OrtSyncStreamImpl instance. nullptr if the execution provider is not stream aware.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(CreateSyncStreamForDevice, _In_ OrtEp* this_ptr,
|
||||
_In_ const OrtMemoryDevice* memory_device,
|
||||
_Outptr_ OrtSyncStreamImpl** stream);
|
||||
|
||||
/** \brief Get a string with details about the EP stack used to produce a compiled model.
|
||||
*
|
||||
* This function gets a compatibility information string that contains details about the execution provider
|
||||
* used to compile a given model. This string can later be used with ValidateCompiledModelCompatibilityInfo
|
||||
* to determine if a compiled model is compatible with the EP.
|
||||
*
|
||||
* The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEp instance.
|
||||
* \param[in] graph The OrtGraph instance for which to generate compatibility information.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(const char*, GetCompiledModelCompatibilityInfo, _In_ OrtEp* this_ptr,
|
||||
_In_ const OrtGraph* graph);
|
||||
};
|
||||
|
||||
/** \brief The function signature that ORT will call to create OrtEpFactory instances.
|
||||
*
|
||||
* This must be available in a function called 'CreateEpFactories' in the execution provider library.
|
||||
*
|
||||
* \param[in] registered_name The name the execution library is registered with by RegisterExecutionProviderLibrary
|
||||
* \param[in] ort_api_base The OrtApiBase instance that is used by the factory to get the OrtApi instance for the
|
||||
* version of ORT that the library was compiled against.
|
||||
* \param[in] default_logger The default ORT logger that can be used for logging outside of an inference session.
|
||||
* \param[in,out] factories The implementation should create and add OrtEpFactory instances to this
|
||||
* pre-allocated array.
|
||||
* i.e. usage is `factories[0] = new MyEpFactory();`
|
||||
* \param[in] max_factories The maximum number of OrtEpFactory instances that can be added to `factories`.
|
||||
* Current default is to allow 4 factories. This can be increased in the future if needed.
|
||||
* \param[out] num_factories The number of OrtEpFactory instances created by the factory and added to `factories`.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.22.
|
||||
*/
|
||||
typedef OrtStatus* (*CreateEpApiFactoriesFn)(_In_ const char* registered_name, _In_ const OrtApiBase* ort_api_base,
|
||||
_In_ const OrtLogger* default_logger,
|
||||
_Inout_ OrtEpFactory** factories, _In_ size_t max_factories,
|
||||
_Out_ size_t* num_factories);
|
||||
|
||||
/** \brief The function signature that ORT will call to release an OrtEpFactory instance.
|
||||
*
|
||||
* This must be available in a function called 'ReleaseEpFactory' in the execution provider library.
|
||||
*
|
||||
* \param[in] factory The OrtEpFactory instance to release.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.22.
|
||||
*/
|
||||
typedef OrtStatus* (*ReleaseEpApiFactoryFn)(_In_ OrtEpFactory* factory);
|
||||
|
||||
/**
|
||||
* \brief The OrtEpFactory provides functions to create and manage execution providers.
|
||||
* \since Version 1.22.
|
||||
*/
|
||||
struct OrtEpFactory {
|
||||
/** \brief The ONNX Runtime version the execution provider was compiled with.
|
||||
*
|
||||
* Implementation should set to ORT_API_VERSION.
|
||||
* ORT will use this to ensure it does not call functions that were not available when the library was compiled.
|
||||
*
|
||||
* \since Version 1.22.
|
||||
*/
|
||||
uint32_t ort_version_supported;
|
||||
|
||||
/** \brief Get the name of the execution provider that the factory creates.
|
||||
*
|
||||
* The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEpFactory instance.
|
||||
* \return The name of the execution provider the factory creates.
|
||||
*
|
||||
* \since Version 1.22.
|
||||
*/
|
||||
ORT_API_T(const char*, GetName, const OrtEpFactory* this_ptr);
|
||||
|
||||
/** \brief Get the name of vendor who owns the execution provider that the factory creates.
|
||||
*
|
||||
* The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEpFactory instance.
|
||||
* \return vendor The vendor name of the execution provider the factory creates.
|
||||
*
|
||||
* \since Version 1.22.
|
||||
*/
|
||||
ORT_API_T(const char*, GetVendor, const OrtEpFactory* this_ptr); // return EP vendor
|
||||
|
||||
/** \brief Get information from the execution provider about OrtHardwareDevice support.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEpFactory instance.
|
||||
* Non-const as the factory is passed through to the CreateEp call via the OrtEpDevice.
|
||||
* \param[in] devices The OrtHardwareDevice instances that are available.
|
||||
* \param[in] num_devices The number of OrtHardwareDevice instances.
|
||||
* \param[out] ep_devices OrtEpDevice instances for each OrtHardwareDevice that the EP can use.
|
||||
* The implementation should call OrtEpApi::CreateEpDevice to create, and add the OrtEpDevice
|
||||
* instances to this pre-allocated array. ORT will take ownership of the values returned.
|
||||
* i.e. usage is `ep_devices[0] = <ptr to OrtEpDevice created with OrtEpApi::CreateEpDevice>;`
|
||||
* \param[in] max_ep_devices The maximum number of OrtEpDevices that can be added to ep_devices.
|
||||
* Current default is 8. This can be increased if needed.
|
||||
* \param[out] num_ep_devices The number of EP devices added to ep_devices.
|
||||
* \return true if the factory can create an execution provider that uses `device`.
|
||||
*
|
||||
* \since Version 1.22.
|
||||
*/
|
||||
ORT_API2_STATUS(GetSupportedDevices, _In_ OrtEpFactory* this_ptr,
|
||||
_In_reads_(num_devices) const OrtHardwareDevice* const* devices,
|
||||
_In_ size_t num_devices,
|
||||
_Inout_ OrtEpDevice** ep_devices,
|
||||
_In_ size_t max_ep_devices,
|
||||
_Out_ size_t* num_ep_devices);
|
||||
|
||||
/** \brief Function to create an OrtEp instance for use in a Session.
|
||||
*
|
||||
* ORT will call ReleaseEp to release the instance when it is no longer needed.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEpFactory instance.
|
||||
* \param[in] devices The OrtHardwareDevice instances that the execution provider was selected to use.
|
||||
* May be a subset of the OrtHardwareDevice instances that the execution provider's factory
|
||||
* set as supported in the call to OrtEpFactory::GetSupportedDevices.
|
||||
* \param[in] ep_metadata_pairs Execution provider metadata that was provided to OrtEpApi::CreateEpDevice, for each
|
||||
* device.
|
||||
* \param[in] num_devices The number of devices the execution provider was selected for.
|
||||
* \param[in] session_options The OrtSessionOptions instance that contains the configuration options for the
|
||||
* session. This will include ep_options from GetSupportedDevices as well as any
|
||||
* user provided overrides.
|
||||
* Execution provider options will have been added with a prefix of 'ep.[ep name].'.
|
||||
* The OrtSessionOptions instance will NOT be valid after this call and should not be
|
||||
* stored for later use.
|
||||
* \param[in] logger The OrtLogger instance for the session that the execution provider should use for logging.
|
||||
* \param[out] ep The OrtEp instance created by the factory.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.22.
|
||||
*/
|
||||
ORT_API2_STATUS(CreateEp, _In_ OrtEpFactory* this_ptr,
|
||||
_In_reads_(num_devices) const OrtHardwareDevice* const* devices,
|
||||
_In_reads_(num_devices) const OrtKeyValuePairs* const* ep_metadata_pairs,
|
||||
_In_ size_t num_devices,
|
||||
_In_ const OrtSessionOptions* session_options,
|
||||
_In_ const OrtLogger* logger, _Outptr_ OrtEp** ep);
|
||||
|
||||
/** \brief Release the OrtEp instance.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEpFactory instance.
|
||||
* \param[in] ep The OrtEp instance to release.
|
||||
*
|
||||
* \since Version 1.22.
|
||||
*/
|
||||
ORT_API_T(void, ReleaseEp, OrtEpFactory* this_ptr, struct OrtEp* ep);
|
||||
|
||||
/** \brief Get the vendor id who owns the execution provider that the factory creates.
|
||||
*
|
||||
* This is typically the PCI vendor ID. See https://pcisig.com/membership/member-companies
|
||||
*
|
||||
* \param[in] this_ptr The OrtEpFactory instance.
|
||||
* \return vendor_id The vendor ID of the execution provider the factory creates.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(uint32_t, GetVendorId, const OrtEpFactory* this_ptr);
|
||||
|
||||
/** \brief Get the version of the execution provider that the factory creates.
|
||||
*
|
||||
* The version string should adhere to the Semantic Versioning 2.0 specification
|
||||
* (https://github.com/semver/semver/blob/v2.0.0/semver.md).
|
||||
*
|
||||
* The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEpFactory instance.
|
||||
* \return The execution provider version string.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(const char*, GetVersion, _In_ const OrtEpFactory* this_ptr);
|
||||
|
||||
/** \brief Validate the compatibility of a compiled model with the execution provider factory for one or more devices.
|
||||
*
|
||||
* Given a compatibility info string produced during model compilation, the EP factory should determine whether the
|
||||
* compiled model is compatible with the EP factory when targeting the provided hardware devices. All devices provided
|
||||
* must belong to the same execution provider instance that this factory creates.
|
||||
*
|
||||
* The EP factory implementation should consider the set of devices (e.g., multi-adapter or multi-GPU scenarios) when
|
||||
* evaluating compatibility and set `model_compatibility` accordingly.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEpFactory instance.
|
||||
* \param[in] devices Array of OrtHardwareDevice pointers that the EP would run on. All must map to this EP.
|
||||
* \param[in] num_devices Number of entries in `devices`.
|
||||
* \param[in] compatibility_info The compatibility information string produced when the model was compiled.
|
||||
* \param[out] model_compatibility OrtCompiledModelCompatibility value describing the compatibility of the model with the EP.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(ValidateCompiledModelCompatibilityInfo, _In_ OrtEpFactory* this_ptr,
|
||||
_In_reads_(num_devices) const OrtHardwareDevice* const* devices,
|
||||
_In_ size_t num_devices,
|
||||
_In_ const char* compatibility_info,
|
||||
_Out_ OrtCompiledModelCompatibility* model_compatibility);
|
||||
|
||||
/** \brief Create an OrtAllocator that can be shared across sessions for the given OrtMemoryInfo.
|
||||
*
|
||||
* The factory that creates the EP is responsible for providing the allocators required by the EP.
|
||||
* The OrtMemoryInfo instance will match one of the values set in the OrtEpDevice using EpDevice_AddAllocatorInfo.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEpFactory instance.
|
||||
* \param[in] memory_info The OrtMemoryInfo to create the allocator for. May be nullptr.
|
||||
* \param[in] allocator_options Optional key-value pairs for allocator options, can be nullptr.
|
||||
* \param[out] allocator The created OrtAllocator instance. Set to nullptr if the default CPU allocator is used.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(CreateAllocator, _In_ OrtEpFactory* this_ptr,
|
||||
_In_ const OrtMemoryInfo* memory_info,
|
||||
_In_opt_ const OrtKeyValuePairs* allocator_options,
|
||||
_Outptr_result_maybenull_ OrtAllocator** allocator);
|
||||
|
||||
/** \brief Release an OrtAllocator created by the factory.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(void, ReleaseAllocator, _In_ OrtEpFactory* this_ptr, _In_ OrtAllocator* allocator);
|
||||
|
||||
/** \brief Create an OrtDataTransferImpl instance for the factory.
|
||||
*
|
||||
* This is used to create an IDataTransfer implementation that can be used to copy data between devices
|
||||
* that the execution provider supports.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEpFactory instance.
|
||||
* \param[out] data_transfer The created OrtDataTransferImpl instance. Set to nullptr if not required.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(CreateDataTransfer, _In_ OrtEpFactory* this_ptr,
|
||||
_Outptr_result_maybenull_ OrtDataTransferImpl** data_transfer);
|
||||
|
||||
/** \brief Check if execution providers created by the factory are stream aware.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEpFactory instance.
|
||||
* \return True if the factory creates execution providers that are stream aware and it implements CreateSyncStreamForDevice.
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API_T(bool, IsStreamAware, _In_ const OrtEpFactory* this_ptr);
|
||||
|
||||
/** \brief Create a synchronization stream for the given memory device.
|
||||
*
|
||||
* This is used to create a synchronization stream for the memory device that can be used for operations outside of
|
||||
* a session.
|
||||
*
|
||||
* \param[in] this_ptr The OrtEpFactory instance.
|
||||
* \param[in] memory_device The OrtMemoryDevice to create the synchronization stream for.
|
||||
* \param[in] stream_options Options for stream creation. May be nullptr.
|
||||
* \param[out] stream The created OrtSyncStreamImpl instance. nullptr if the execution provider is not stream aware.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
* \since Version 1.23.
|
||||
*/
|
||||
ORT_API2_STATUS(CreateSyncStreamForDevice, _In_ OrtEpFactory* this_ptr,
|
||||
_In_ const OrtMemoryDevice* memory_device,
|
||||
_In_opt_ const OrtKeyValuePairs* stream_options,
|
||||
_Outptr_ OrtSyncStreamImpl** stream);
|
||||
};
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
|
||||
// This file contains well-known keys for OrtEpDevice EP metadata entries.
|
||||
// It does NOT specify all available metadata keys.
|
||||
|
||||
// Key for the execution provider version string. This should be available for all plugin EPs.
|
||||
static const char* const kOrtEpDevice_EpMetadataKey_Version = "version";
|
||||
|
||||
// Prefix for execution provider compatibility information stored in model metadata.
|
||||
// Used when generating EP context models to store compatibility strings for each EP.
|
||||
// Full key format: "ep_compatibility_info.<EP_TYPE>"
|
||||
static const char* const kOrtModelMetadata_EpCompatibilityInfoPrefix = "ep_compatibility_info.";
|
||||
|
||||
// Key for the execution provider library path (for dynamically loaded EPs)
|
||||
static const char* const kOrtEpDevice_EpMetadataKey_LibraryPath = "library_path";
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading…
Reference in New Issue