From df7192f6d276828a0cd2a2060979d947eecc0748 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Wed, 6 Aug 2025 09:54:18 -0700 Subject: [PATCH 1/7] cpp introduction of sourceloc type --- ast_canopy/cpp/CMakeLists.txt | 5 ++- .../cpp/include/ast_canopy/ast_canopy.hpp | 38 +++++++++++-------- .../include/ast_canopy/source_location.hpp | 30 +++++++++++++++ ast_canopy/cpp/src/ast_canopy.cpp | 4 +- ast_canopy/cpp/src/constexpr_vardecl.cpp | 3 +- ast_canopy/cpp/src/decl.cpp | 30 +++++++++++++++ ast_canopy/cpp/src/detail/matchers.hpp | 2 +- ast_canopy/cpp/src/enum.cpp | 2 +- ast_canopy/cpp/src/field.cpp | 3 +- ast_canopy/cpp/src/param_var.cpp | 4 +- ast_canopy/cpp/src/source_location.cpp | 18 +++++++++ 11 files changed, 115 insertions(+), 24 deletions(-) create mode 100644 ast_canopy/cpp/include/ast_canopy/source_location.hpp create mode 100644 ast_canopy/cpp/src/decl.cpp create mode 100644 ast_canopy/cpp/src/source_location.cpp diff --git a/ast_canopy/cpp/CMakeLists.txt b/ast_canopy/cpp/CMakeLists.txt index c3110343..a72d06c0 100644 --- a/ast_canopy/cpp/CMakeLists.txt +++ b/ast_canopy/cpp/CMakeLists.txt @@ -15,6 +15,7 @@ set(CMAKE_CXX_EXTENSIONS OFF) # Define the sources to compile for astcanopy here. add_library(astcanopy SHARED src/ast_canopy.cpp + src/decl.cpp src/enum.cpp src/class_template.cpp src/field.cpp @@ -28,6 +29,7 @@ add_library(astcanopy SHARED src/type.cpp src/typedef.cpp src/constexpr_vardecl.cpp + src/source_location.cpp src/detail/matchers/function_matcher.cpp src/detail/matchers/record_matcher.cpp src/detail/matchers/typedef_matcher.cpp @@ -44,7 +46,8 @@ target_sources(astcanopy TYPE HEADERS BASE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/include FILES include/ast_canopy/ast_canopy.hpp - include/ast_canopy/error.hpp) + include/ast_canopy/error.hpp + include/ast_canopy/source_location.hpp) # astcanopy shared lib target find_package(Clang REQUIRED) diff --git a/ast_canopy/cpp/include/ast_canopy/ast_canopy.hpp b/ast_canopy/cpp/include/ast_canopy/ast_canopy.hpp index 7a41fee0..9f572c9c 100644 --- a/ast_canopy/cpp/include/ast_canopy/ast_canopy.hpp +++ b/ast_canopy/cpp/include/ast_canopy/ast_canopy.hpp @@ -16,6 +16,7 @@ #include #include +#include namespace ast_canopy { @@ -38,10 +39,16 @@ enum class template_param_kind { type, non_type, template_ }; enum class access_kind { public_, protected_, private_ }; -struct Enum { +struct Decl { + SourceLocation source_location; + Decl(); + Decl(const clang::Decl *decl); +}; + +struct Enum : public Decl { Enum(const std::string &name, const std::vector &enumerators, const std::vector &enumerator_values) - : name(name), enumerators(enumerators), + : Decl(), name(name), enumerators(enumerators), enumerator_values(enumerator_values) {} Enum(const clang::EnumDecl *); @@ -67,7 +74,7 @@ struct Type { bool _is_left_reference; }; -struct ConstExprVar { +struct ConstExprVar : public Decl { ConstExprVar() = default; ConstExprVar(const clang::VarDecl *VD); @@ -76,18 +83,19 @@ struct ConstExprVar { std::string value; }; -struct Field { +struct Field : public Decl { Field(const clang::FieldDecl *FD, const clang::AccessSpecifier &AS); Field(const std::string &name, const Type &type, const access_kind &access) - : name(name), type(type), access(access) {}; + : Decl(), name(name), type(type), access(access) {}; std::string name; Type type; access_kind access; }; -struct ParamVar { - ParamVar(std::string name, Type type) : name(std::move(name)), type(type) {} +struct ParamVar : public Decl { + ParamVar(std::string name, Type type) + : Decl(), name(std::move(name)), type(type) {} ParamVar(const clang::ParmVarDecl *PVD); std::string name; @@ -106,9 +114,9 @@ struct Template { virtual ~Template() = default; }; -struct TemplateParam { +struct TemplateParam : public Decl { TemplateParam(const std::string &name, template_param_kind kind, Type type) - : name(name), kind(kind), type(type) {} + : Decl(), name(name), kind(kind), type(type) {} TemplateParam(const clang::TemplateTypeParmDecl *); TemplateParam(const clang::NonTypeTemplateParmDecl *); TemplateParam(const clang::TemplateTemplateParmDecl *); @@ -118,11 +126,11 @@ struct TemplateParam { Type type; }; -struct Function { +struct Function : public Decl { Function(const std::string &name, const Type &return_type, const std::vector ¶ms, const execution_space &exec_space) - : name(name), return_type(return_type), params(params), + : Decl(), name(name), return_type(return_type), params(params), exec_space(exec_space) {} Function(const clang::FunctionDecl *); @@ -163,7 +171,7 @@ enum class RecordAncestor { ANCESTOR_IS_NOT_TEMPLATE, }; -struct Record { +struct Record : public Decl { Record(const std::string &name, const std::vector &fields, const std::vector &methods, const std::vector &templated_methods, @@ -171,7 +179,7 @@ struct Record { const std::vector &nested_class_templates, const std::size_t &sizeof_, const std::size_t &alignof_, const std::string &source_range) - : name(name), fields(fields), methods(methods), + : Decl(), name(name), fields(fields), methods(methods), templated_methods(templated_methods), nested_records(nested_records), nested_class_templates(nested_class_templates), sizeof_(sizeof_), alignof_(alignof_), source_range(source_range) {} @@ -197,9 +205,9 @@ struct ClassTemplate : public Template { Record record; }; -struct Typedef { +struct Typedef : public Decl { Typedef(const std::string &name, const std::string &underlying_name) - : name(name), underlying_name(underlying_name) {} + : Decl(), name(name), underlying_name(underlying_name) {} Typedef(const clang::TypedefDecl *, std::unordered_map *); diff --git a/ast_canopy/cpp/include/ast_canopy/source_location.hpp b/ast_canopy/cpp/include/ast_canopy/source_location.hpp new file mode 100644 index 00000000..49e1cc50 --- /dev/null +++ b/ast_canopy/cpp/include/ast_canopy/source_location.hpp @@ -0,0 +1,30 @@ +// clang-format off +// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// clang-format on + +#pragma once + +#include + +namespace ast_canopy { + +class SourceLocation { +public: + SourceLocation(); + SourceLocation(const std::string &file_name, const unsigned int line, + const unsigned int column, const bool is_valid); + + bool is_valid() const { return _is_valid; } + std::string file_name() const { return _file_name; } + unsigned int line() const { return _line; } + unsigned int column() const { return _column; } + +private: + bool _is_valid; + std::string _file_name; + unsigned int _line; + unsigned int _column; +}; + +}; // namespace ast_canopy diff --git a/ast_canopy/cpp/src/ast_canopy.cpp b/ast_canopy/cpp/src/ast_canopy.cpp index 67d5fc8a..10142f0e 100644 --- a/ast_canopy/cpp/src/ast_canopy.cpp +++ b/ast_canopy/cpp/src/ast_canopy.cpp @@ -47,7 +47,7 @@ class AstCanopyDiagnosticsConsumer : public DiagnosticConsumer { // Add source location if available if (Info.hasSourceManager() && Info.getLocation().isValid()) { const SourceManager &SM = Info.getSourceManager(); - SourceLocation loc = Info.getLocation(); + clang::SourceLocation loc = Info.getLocation(); PresumedLoc PLoc = SM.getPresumedLoc(loc); if (PLoc.isValid()) { verbose_message += PLoc.getFilename(); @@ -67,7 +67,7 @@ class AstCanopyDiagnosticsConsumer : public DiagnosticConsumer { /** * @brief Return the source filename of the declaration. */ -std::string source_filename_from_decl(const Decl *D) { +std::string source_filename_from_decl(const clang::Decl *D) { const ASTContext &ast_context = D->getASTContext(); const SourceManager &sm = ast_context.getSourceManager(); const StringRef file_name_ref = sm.getFilename(D->getLocation()); diff --git a/ast_canopy/cpp/src/constexpr_vardecl.cpp b/ast_canopy/cpp/src/constexpr_vardecl.cpp index 7e25b3cb..003090ac 100644 --- a/ast_canopy/cpp/src/constexpr_vardecl.cpp +++ b/ast_canopy/cpp/src/constexpr_vardecl.cpp @@ -9,7 +9,8 @@ namespace ast_canopy { ConstExprVar::ConstExprVar(const clang::VarDecl *VD) - : type_(VD->getType(), VD->getASTContext()), name(VD->getNameAsString()) { + : Decl(VD), type_(VD->getType(), VD->getASTContext()), + name(VD->getNameAsString()) { if (!VD->isConstexpr()) { throw std::runtime_error("Not a constexpr variable"); } diff --git a/ast_canopy/cpp/src/decl.cpp b/ast_canopy/cpp/src/decl.cpp new file mode 100644 index 00000000..75939c9e --- /dev/null +++ b/ast_canopy/cpp/src/decl.cpp @@ -0,0 +1,30 @@ +// clang-format off +// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// clang-format on + +#include +#include + +#include +#include + +namespace ast_canopy { + +Decl::Decl() : source_location(SourceLocation()) {} + +Decl::Decl(const clang::Decl *decl) { + clang::SourceLocation loc = clang::SourceLocation(decl->getLocation()); + bool is_valid = loc.isValid(); + if (is_valid) { + clang::SourceManager &SM = decl->getASTContext().getSourceManager(); + unsigned int row = SM.getSpellingColumnNumber(loc); + unsigned int col = SM.getSpellingColumnNumber(loc); + llvm::StringRef file_name = SM.getFilename(loc); + source_location = SourceLocation(file_name.str(), row, col, is_valid); + } else { + source_location = SourceLocation("", 0, 0, is_valid); + } +} + +} // namespace ast_canopy diff --git a/ast_canopy/cpp/src/detail/matchers.hpp b/ast_canopy/cpp/src/detail/matchers.hpp index 103ebdc6..31fe6fa0 100644 --- a/ast_canopy/cpp/src/detail/matchers.hpp +++ b/ast_canopy/cpp/src/detail/matchers.hpp @@ -37,7 +37,7 @@ struct vardecl_matcher_payload { std::optional var; }; -std::string source_filename_from_decl(const Decl *); +std::string source_filename_from_decl(const clang::Decl *); class FunctionCallback : public MatchFinder::MatchCallback { public: diff --git a/ast_canopy/cpp/src/enum.cpp b/ast_canopy/cpp/src/enum.cpp index 8f9519a7..122fc222 100644 --- a/ast_canopy/cpp/src/enum.cpp +++ b/ast_canopy/cpp/src/enum.cpp @@ -7,7 +7,7 @@ namespace ast_canopy { -Enum::Enum(const clang::EnumDecl *ED) : name(ED->getNameAsString()) { +Enum::Enum(const clang::EnumDecl *ED) : Decl(ED), name(ED->getNameAsString()) { for (const auto *enumerator : ED->enumerators()) { enumerators.push_back(enumerator->getNameAsString()); diff --git a/ast_canopy/cpp/src/field.cpp b/ast_canopy/cpp/src/field.cpp index 6596f4e3..8cea52ba 100644 --- a/ast_canopy/cpp/src/field.cpp +++ b/ast_canopy/cpp/src/field.cpp @@ -12,7 +12,8 @@ namespace ast_canopy { Field::Field(const clang::FieldDecl *FD, const clang::AccessSpecifier &AS) - : name(FD->getNameAsString()), type(FD->getType(), FD->getASTContext()) { + : Decl(FD), name(FD->getNameAsString()), + type(FD->getType(), FD->getASTContext()) { if (AS == clang::AccessSpecifier::AS_public) access = access_kind::public_; else if (AS == clang::AccessSpecifier::AS_protected) diff --git a/ast_canopy/cpp/src/param_var.cpp b/ast_canopy/cpp/src/param_var.cpp index 17c3c605..32788044 100644 --- a/ast_canopy/cpp/src/param_var.cpp +++ b/ast_canopy/cpp/src/param_var.cpp @@ -7,7 +7,7 @@ namespace ast_canopy { ParamVar::ParamVar(const clang::ParmVarDecl *PVD) - : name(PVD->getNameAsString()), type(PVD->getType(), PVD->getASTContext()) { -} + : Decl(PVD), name(PVD->getNameAsString()), + type(PVD->getType(), PVD->getASTContext()) {} } // namespace ast_canopy diff --git a/ast_canopy/cpp/src/source_location.cpp b/ast_canopy/cpp/src/source_location.cpp new file mode 100644 index 00000000..ccbef6e7 --- /dev/null +++ b/ast_canopy/cpp/src/source_location.cpp @@ -0,0 +1,18 @@ +// clang-format off +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// clang-format on + +#include + +namespace ast_canopy { + +SourceLocation::SourceLocation() : _is_valid(false) {} + +SourceLocation::SourceLocation(const std::string &file_name, + const unsigned int line, + const unsigned int column, const bool is_valid) + : _file_name(file_name), _line(line), _column(column), _is_valid(is_valid) { +} + +} // namespace ast_canopy From 515beb6318b0fa69f6d6f1563fc685ac9fda2ebe Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Wed, 6 Aug 2025 12:50:41 -0700 Subject: [PATCH 2/7] Surfacing source location line num and column num --- ast_canopy/ast_canopy/decl.py | 47 +++++++++++++++---- ast_canopy/ast_canopy/pylibastcanopy.cpp | 34 ++++++++++---- .../cpp/include/ast_canopy/ast_canopy.hpp | 7 +-- ast_canopy/cpp/src/class_template.cpp | 2 +- ast_canopy/cpp/src/decl.cpp | 5 +- ast_canopy/cpp/src/function.cpp | 2 +- ast_canopy/cpp/src/function_template.cpp | 2 +- ast_canopy/cpp/src/record.cpp | 2 +- ast_canopy/tests/data/sample_source_loc.cu | 18 +++++++ ast_canopy/tests/test_sample_source_loc.py | 47 +++++++++++++++++++ 10 files changed, 139 insertions(+), 27 deletions(-) create mode 100644 ast_canopy/tests/data/sample_source_loc.cu create mode 100644 ast_canopy/tests/test_sample_source_loc.py diff --git a/ast_canopy/ast_canopy/decl.py b/ast_canopy/ast_canopy/decl.py index a2fa24d0..d828db5b 100644 --- a/ast_canopy/ast_canopy/decl.py +++ b/ast_canopy/ast_canopy/decl.py @@ -58,7 +58,12 @@ } -class Function: +class Decl: + def __init__(self, source_location: bindings.SourceLocation): + self.source_location = source_location + + +class Function(Decl): """ Represents a C++ function. @@ -75,7 +80,9 @@ def __init__( is_constexpr: bool, mangled_name: str, parse_entry_point: str, + source_location: bindings.SourceLocation, ): + super().__init__(source_location) self.name = name self.return_type = return_type self.params = params @@ -163,6 +170,7 @@ def from_c_obj(cls, c_obj: bindings.Function, parse_entry_point: str): c_obj.is_constexpr, c_obj.mangled_name, parse_entry_point, + c_obj.source_location, ) @@ -172,15 +180,17 @@ def __init__(self, template_parameters, num_min_required_args): self.num_min_required_args = num_min_required_args -class FunctionTemplate(Template): +class FunctionTemplate(Decl, Template): def __init__( self, template_parameters: list[bindings.TemplateParam], num_min_required_args: int, function: Function, parse_entry_point: str, + source_location: bindings.SourceLocation, ): - super().__init__(template_parameters, num_min_required_args) + Decl.__init__(self, source_location) + Template.__init__(self, template_parameters, num_min_required_args) self.function = function self.parse_entry_point = parse_entry_point @@ -194,6 +204,7 @@ def from_c_obj( c_obj.num_min_required_args, Function.from_c_obj(c_obj.function, parse_entry_point), parse_entry_point, + c_obj.source_location, ) def instantiate(self, **kwargs): @@ -213,6 +224,7 @@ def __init__( is_move_constructor: bool, mangled_name: str, parse_entry_point: str, + source_location: bindings.SourceLocation, ): super().__init__( name, @@ -222,6 +234,7 @@ def __init__( is_constexpr, mangled_name, parse_entry_point, + source_location, ) self.kind = kind self.is_move_constructor = is_move_constructor @@ -252,6 +265,7 @@ def from_c_obj(cls, c_obj: bindings.Method, parse_entry_point: str): c_obj.is_move_constructor(), c_obj.mangled_name, parse_entry_point, + c_obj.source_location, ) @@ -273,7 +287,7 @@ def decl_name(self): return self.name -class Struct: +class Struct(Decl): def __init__( self, name: str, @@ -285,7 +299,9 @@ def __init__( sizeof_: int, alignof_: int, parse_entry_point: str, + source_location: bindings.SourceLocation, ): + super().__init__(source_location) self.name = name self.fields = fields self.methods = methods @@ -330,6 +346,7 @@ def from_c_obj(cls, c_obj: bindings.Record, parse_entry_point: str): c_obj.sizeof_, c_obj.alignof_, parse_entry_point, + c_obj.source_location, ) @@ -354,18 +371,21 @@ def from_c_obj(cls, c_obj: bindings.Record, parse_entry_point: str): c_obj.sizeof_, c_obj.alignof_, parse_entry_point, + c_obj.source_location, ) -class ClassTemplate(Template): +class ClassTemplate(Decl, Template): def __init__( self, record: TemplatedStruct, template_parameters: list[bindings.TemplateParam], num_min_required_args: int, parse_entry_point: str, + source_location: bindings.SourceLocation, ): - super().__init__(template_parameters, num_min_required_args) + Decl.__init__(self, source_location) + Template.__init__(self, template_parameters, num_min_required_args) self.record = record self.parse_entry_point = parse_entry_point @@ -377,6 +397,7 @@ def from_c_obj(cls, c_obj: bindings.ClassTemplate, parse_entry_point: str): c_obj.template_parameters, c_obj.num_min_required_args, parse_entry_point, + c_obj.source_location, ) def instantiate(self, **kwargs): @@ -384,15 +405,23 @@ def instantiate(self, **kwargs): return tstruct.instantiate(**kwargs) -class ConstExprVar: - def __init__(self, name: str, type_: bindings.Type, value_serialized: str): +class ConstExprVar(Decl): + def __init__( + self, + name: str, + type_: bindings.Type, + value_serialized: str, + source_location: bindings.SourceLocation, + ): + super().__init__(source_location) + self.name = name self.type_ = type_ self.value_serialized = value_serialized @classmethod def from_c_obj(cls, c_obj: bindings.ConstExprVar): - return cls(c_obj.name, c_obj.type_, c_obj.value) + return cls(c_obj.name, c_obj.type_, c_obj.value, c_obj.source_location) @property def value(self): diff --git a/ast_canopy/ast_canopy/pylibastcanopy.cpp b/ast_canopy/ast_canopy/pylibastcanopy.cpp index ea2f2a00..b981dc7c 100644 --- a/ast_canopy/ast_canopy/pylibastcanopy.cpp +++ b/ast_canopy/ast_canopy/pylibastcanopy.cpp @@ -43,7 +43,21 @@ PYBIND11_MODULE(pylibastcanopy, m) { .value("protected_", access_kind::protected_) .value("private_", access_kind::private_); - py::class_(m, "Enum") + py::class_(m, "SourceLocation") + .def(py::init<>()) + .def(py::init()) + .def_property_readonly("file_name", &SourceLocation::file_name) + .def_property_readonly("line", &SourceLocation::line) + .def_property_readonly("column", &SourceLocation::column) + .def_property_readonly("is_valid", &SourceLocation::is_valid); + + py::class_(m, "Decl") + .def(py::init()) + .def(py::init()) + .def_readwrite("source_location", &Decl::source_location); + + py::class_(m, "Enum") .def(py::init()) .def_readwrite("name", &Enum::name) .def_readwrite("enumerators", &Enum::enumerators) @@ -82,13 +96,13 @@ PYBIND11_MODULE(pylibastcanopy, m) { t[2].cast(), t[3].cast()}; })); - py::class_(m, "ConstExprVar") + py::class_(m, "ConstExprVar") .def(py::init<>()) .def_readwrite("type_", &ConstExprVar::type_) .def_readwrite("name", &ConstExprVar::name) .def_readwrite("value", &ConstExprVar::value); - py::class_(m, "Field") + py::class_(m, "Field") .def_readwrite("name", &Field::name) .def_readwrite("type_", &Field::type) .def_readwrite("access", &Field::access) @@ -107,7 +121,7 @@ PYBIND11_MODULE(pylibastcanopy, m) { t[2].cast()}; })); - py::class_(m, "ParamVar") + py::class_(m, "ParamVar") .def(py::init()) .def_readwrite("name", &ParamVar::name) .def_readwrite("type_", &ParamVar::type) @@ -124,7 +138,7 @@ PYBIND11_MODULE(pylibastcanopy, m) { return ParamVar{t[0].cast(), t[1].cast()}; })); - py::class_(m, "TemplateParam") + py::class_(m, "TemplateParam") .def_readwrite("name", &TemplateParam::name) .def_readwrite("type_", &TemplateParam::type) .def_readwrite("kind", &TemplateParam::kind) @@ -145,7 +159,7 @@ PYBIND11_MODULE(pylibastcanopy, m) { t[2].cast()}; })); - py::class_(m, "Function") + py::class_(m, "Function") .def_readwrite("name", &Function::name) .def_readwrite("return_type", &Function::return_type) .def_readwrite("params", &Function::params) @@ -183,7 +197,7 @@ PYBIND11_MODULE(pylibastcanopy, m) { t[1].cast()}; })); - py::class_(m, "FunctionTemplate") + py::class_(m, "FunctionTemplate") .def_readwrite("function", &FunctionTemplate::function) .def_readwrite("num_min_required_args", &FunctionTemplate::num_min_required_args) @@ -202,7 +216,7 @@ PYBIND11_MODULE(pylibastcanopy, m) { t[1].cast()}; })); - py::class_(m, "ClassTemplate") + py::class_(m, "ClassTemplate") .def_readwrite("num_min_required_args", &ClassTemplate::num_min_required_args) .def_readwrite("record", &ClassTemplate::record); @@ -224,7 +238,7 @@ PYBIND11_MODULE(pylibastcanopy, m) { t[1].cast()}; })); - py::class_(m, "Record") + py::class_(m, "Record") .def_readwrite("name", &Record::name) .def_readwrite("fields", &Record::fields) .def_readwrite("methods", &Record::methods) @@ -254,7 +268,7 @@ PYBIND11_MODULE(pylibastcanopy, m) { t[8].cast()}; })); - py::class_(m, "Typedef") + py::class_(m, "Typedef") .def_readwrite("name", &Typedef::name) .def_readwrite("underlying_name", &Typedef::underlying_name) .def(py::pickle( diff --git a/ast_canopy/cpp/include/ast_canopy/ast_canopy.hpp b/ast_canopy/cpp/include/ast_canopy/ast_canopy.hpp index 9f572c9c..6ed6ec1d 100644 --- a/ast_canopy/cpp/include/ast_canopy/ast_canopy.hpp +++ b/ast_canopy/cpp/include/ast_canopy/ast_canopy.hpp @@ -40,9 +40,10 @@ enum class template_param_kind { type, non_type, template_ }; enum class access_kind { public_, protected_, private_ }; struct Decl { - SourceLocation source_location; Decl(); Decl(const clang::Decl *decl); + + SourceLocation source_location; }; struct Enum : public Decl { @@ -142,7 +143,7 @@ struct Function : public Decl { std::string mangled_name; }; -struct FunctionTemplate : public Template { +struct FunctionTemplate : public Decl, public Template { FunctionTemplate(const std::vector &template_parameters, const std::size_t &num_min_required_args, const Function &function) @@ -200,7 +201,7 @@ struct Record : public Decl { void print(int) const; }; -struct ClassTemplate : public Template { +struct ClassTemplate : public Decl, public Template { ClassTemplate(const clang::ClassTemplateDecl *); Record record; }; diff --git a/ast_canopy/cpp/src/class_template.cpp b/ast_canopy/cpp/src/class_template.cpp index 46257bf2..9122ae58 100644 --- a/ast_canopy/cpp/src/class_template.cpp +++ b/ast_canopy/cpp/src/class_template.cpp @@ -13,6 +13,6 @@ namespace ast_canopy { ClassTemplate::ClassTemplate(const clang::ClassTemplateDecl *CTD) - : Template(CTD->getTemplateParameters()), + : Decl(CTD), Template(CTD->getTemplateParameters()), record(CTD->getTemplatedDecl(), RecordAncestor::ANCESTOR_IS_TEMPLATE) {} } // namespace ast_canopy diff --git a/ast_canopy/cpp/src/decl.cpp b/ast_canopy/cpp/src/decl.cpp index 75939c9e..899437d2 100644 --- a/ast_canopy/cpp/src/decl.cpp +++ b/ast_canopy/cpp/src/decl.cpp @@ -9,6 +9,8 @@ #include #include +#include + namespace ast_canopy { Decl::Decl() : source_location(SourceLocation()) {} @@ -16,9 +18,10 @@ Decl::Decl() : source_location(SourceLocation()) {} Decl::Decl(const clang::Decl *decl) { clang::SourceLocation loc = clang::SourceLocation(decl->getLocation()); bool is_valid = loc.isValid(); + std::cout << "is_valid: " << is_valid << std::endl; if (is_valid) { clang::SourceManager &SM = decl->getASTContext().getSourceManager(); - unsigned int row = SM.getSpellingColumnNumber(loc); + unsigned int row = SM.getSpellingLineNumber(loc); unsigned int col = SM.getSpellingColumnNumber(loc); llvm::StringRef file_name = SM.getFilename(loc); source_location = SourceLocation(file_name.str(), row, col, is_valid); diff --git a/ast_canopy/cpp/src/function.cpp b/ast_canopy/cpp/src/function.cpp index 34f61a2d..f3cde3de 100644 --- a/ast_canopy/cpp/src/function.cpp +++ b/ast_canopy/cpp/src/function.cpp @@ -30,7 +30,7 @@ execution_space get_execution_space(const clang::FunctionDecl *FD) { } Function::Function(const clang::FunctionDecl *FD) - : name(FD->getNameAsString()), + : Decl(FD), name(FD->getNameAsString()), return_type(FD->getReturnType(), FD->getASTContext()), is_constexpr(FD->isConstexpr()) { params.reserve(FD->getNumParams()); diff --git a/ast_canopy/cpp/src/function_template.cpp b/ast_canopy/cpp/src/function_template.cpp index 7720c724..7af7171d 100644 --- a/ast_canopy/cpp/src/function_template.cpp +++ b/ast_canopy/cpp/src/function_template.cpp @@ -16,6 +16,6 @@ namespace ast_canopy { FunctionTemplate::FunctionTemplate(const clang::FunctionTemplateDecl *FTD) - : Template(FTD->getTemplateParameters()), + : Decl(FTD), Template(FTD->getTemplateParameters()), function(FTD->getTemplatedDecl()) {} } // namespace ast_canopy diff --git a/ast_canopy/cpp/src/record.cpp b/ast_canopy/cpp/src/record.cpp index 975e3f9a..067ede28 100644 --- a/ast_canopy/cpp/src/record.cpp +++ b/ast_canopy/cpp/src/record.cpp @@ -20,7 +20,7 @@ std::size_t constexpr INVALID_SIZE_OF = std::numeric_limits::max(); std::size_t constexpr INVALID_ALIGN_OF = std::numeric_limits::max(); -Record::Record(const clang::CXXRecordDecl *RD, RecordAncestor rp) { +Record::Record(const clang::CXXRecordDecl *RD, RecordAncestor rp) : Decl(RD) { using AS = clang::AccessSpecifier; #ifndef NDEBUG diff --git a/ast_canopy/tests/data/sample_source_loc.cu b/ast_canopy/tests/data/sample_source_loc.cu new file mode 100644 index 00000000..f5f2793d --- /dev/null +++ b/ast_canopy/tests/data/sample_source_loc.cu @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. SPDX-License-Identifier: Apache-2.0 + +// clang-format off + +void __device__ __forceinline__ foo() {} // line 6 + +struct Bar { // line 8 + Bar() {} // line 9 +}; + +template +void __device__ __forceinline__ baz() {} // line 13 + +template +struct Bax {}; // line 16 + +// clang-format on diff --git a/ast_canopy/tests/test_sample_source_loc.py b/ast_canopy/tests/test_sample_source_loc.py new file mode 100644 index 00000000..94c0e4de --- /dev/null +++ b/ast_canopy/tests/test_sample_source_loc.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + + +from ast_canopy import parse_declarations_from_source + + +def test_source_location(data_folder): + srcstr = str(data_folder / "sample_source_loc.cu") + + decls = parse_declarations_from_source(srcstr, [srcstr], "sm_80") + + assert len(decls.functions) == 1 + assert len(decls.structs) == 1 + assert len(decls.class_templates) == 1 + assert len(decls.function_templates) == 1 + + foo = decls.functions[0] + assert "sample_source_loc.cu" in foo.source_location.file_name + assert foo.source_location.line == 6 + assert foo.source_location.column == 33 + assert foo.source_location.is_valid + + bar = decls.structs[0] + assert "sample_source_loc.cu" in bar.source_location.file_name + assert bar.source_location.line == 8 + assert bar.source_location.column == 8 + assert bar.source_location.is_valid + + assert len(bar.methods) == 1 + barctor = bar.methods[0] + assert "sample_source_loc.cu" in barctor.source_location.file_name + assert barctor.source_location.line == 9 + assert barctor.source_location.column == 5 + assert barctor.source_location.is_valid + + baz = decls.function_templates[0] + assert "sample_source_loc.cu" in baz.source_location.file_name + assert baz.function.source_location.line == 13 + assert baz.function.source_location.column == 33 + assert baz.function.source_location.is_valid + + bax = decls.class_templates[0] + assert "sample_source_loc.cu" in bax.record.source_location.file_name + assert bax.record.source_location.line == 16 + assert bax.record.source_location.column == 8 + assert bax.record.source_location.is_valid From cf92fec499138fd61085a2d9ad4d89cf4279d664 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Wed, 6 Aug 2025 13:21:16 -0700 Subject: [PATCH 3/7] add more test on enum and typedef decl --- ast_canopy/tests/data/sample_source_loc.cu | 10 +++++++--- ast_canopy/tests/test_sample_source_loc.py | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/ast_canopy/tests/data/sample_source_loc.cu b/ast_canopy/tests/data/sample_source_loc.cu index f5f2793d..eeb9dbf8 100644 --- a/ast_canopy/tests/data/sample_source_loc.cu +++ b/ast_canopy/tests/data/sample_source_loc.cu @@ -9,10 +9,14 @@ struct Bar { // line 8 Bar() {} // line 9 }; -template +template // line 12 void __device__ __forceinline__ baz() {} // line 13 -template -struct Bax {}; // line 16 +template // line 15 +struct Bax {}; // line 16 + +enum class Watermelon {}; // line 18 + +typedef Watermelon Suika; // line 20 // clang-format on diff --git a/ast_canopy/tests/test_sample_source_loc.py b/ast_canopy/tests/test_sample_source_loc.py index 94c0e4de..8931fb22 100644 --- a/ast_canopy/tests/test_sample_source_loc.py +++ b/ast_canopy/tests/test_sample_source_loc.py @@ -36,12 +36,34 @@ def test_source_location(data_folder): baz = decls.function_templates[0] assert "sample_source_loc.cu" in baz.source_location.file_name + assert baz.source_location.line == 12 + assert baz.source_location.column == 0 + assert baz.source_location.is_valid + + assert "sample_source_loc.cu" in baz.function.source_location.file_name assert baz.function.source_location.line == 13 assert baz.function.source_location.column == 33 assert baz.function.source_location.is_valid bax = decls.class_templates[0] + assert "sample_source_loc.cu" in bax.source_location.file_name + assert bax.source_location.line == 15 + assert bax.source_location.column == 0 + assert bax.source_location.is_valid + assert "sample_source_loc.cu" in bax.record.source_location.file_name assert bax.record.source_location.line == 16 assert bax.record.source_location.column == 8 assert bax.record.source_location.is_valid + + watermelon = decls.enum_classes[0] + assert "sample_source_loc.cu" in watermelon.source_location.file_name + assert watermelon.source_location.line == 18 + assert watermelon.source_location.column == 0 + assert watermelon.source_location.is_valid + + suika = decls.typedefs[0] + assert "sample_source_loc.cu" in suika.source_location.file_name + assert suika.source_location.line == 19 + assert suika.source_location.column == 0 + assert suika.source_location.is_valid From b0223e7bf560117306ab381be44e03de7c574cff Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Wed, 6 Aug 2025 13:33:10 -0700 Subject: [PATCH 4/7] moving location extraction logic to source_location impl --- .../include/ast_canopy/source_location.hpp | 5 ++++ ast_canopy/cpp/src/decl.cpp | 13 +---------- ast_canopy/cpp/src/source_location.cpp | 23 +++++++++++++++++++ 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/ast_canopy/cpp/include/ast_canopy/source_location.hpp b/ast_canopy/cpp/include/ast_canopy/source_location.hpp index 49e1cc50..aae84b1e 100644 --- a/ast_canopy/cpp/include/ast_canopy/source_location.hpp +++ b/ast_canopy/cpp/include/ast_canopy/source_location.hpp @@ -7,6 +7,7 @@ #include +#include namespace ast_canopy { class SourceLocation { @@ -27,4 +28,8 @@ class SourceLocation { unsigned int _column; }; +namespace detail { +SourceLocation location_from_decl(const clang::Decl *decl); +} + }; // namespace ast_canopy diff --git a/ast_canopy/cpp/src/decl.cpp b/ast_canopy/cpp/src/decl.cpp index 899437d2..5c76e28f 100644 --- a/ast_canopy/cpp/src/decl.cpp +++ b/ast_canopy/cpp/src/decl.cpp @@ -16,18 +16,7 @@ namespace ast_canopy { Decl::Decl() : source_location(SourceLocation()) {} Decl::Decl(const clang::Decl *decl) { - clang::SourceLocation loc = clang::SourceLocation(decl->getLocation()); - bool is_valid = loc.isValid(); - std::cout << "is_valid: " << is_valid << std::endl; - if (is_valid) { - clang::SourceManager &SM = decl->getASTContext().getSourceManager(); - unsigned int row = SM.getSpellingLineNumber(loc); - unsigned int col = SM.getSpellingColumnNumber(loc); - llvm::StringRef file_name = SM.getFilename(loc); - source_location = SourceLocation(file_name.str(), row, col, is_valid); - } else { - source_location = SourceLocation("", 0, 0, is_valid); - } + source_location = detail::location_from_decl(decl); } } // namespace ast_canopy diff --git a/ast_canopy/cpp/src/source_location.cpp b/ast_canopy/cpp/src/source_location.cpp index ccbef6e7..0d3291f6 100644 --- a/ast_canopy/cpp/src/source_location.cpp +++ b/ast_canopy/cpp/src/source_location.cpp @@ -5,6 +5,9 @@ #include +#include +#include + namespace ast_canopy { SourceLocation::SourceLocation() : _is_valid(false) {} @@ -15,4 +18,24 @@ SourceLocation::SourceLocation(const std::string &file_name, : _file_name(file_name), _line(line), _column(column), _is_valid(is_valid) { } +namespace detail { + +SourceLocation location_from_decl(const clang::Decl *decl) { + ast_canopy::SourceLocation res; + clang::SourceLocation loc = clang::SourceLocation(decl->getLocation()); + bool is_valid = loc.isValid(); + if (is_valid) { + clang::SourceManager &SM = decl->getASTContext().getSourceManager(); + unsigned int row = SM.getSpellingLineNumber(loc); + unsigned int col = SM.getSpellingColumnNumber(loc); + llvm::StringRef file_name = SM.getFilename(loc); + res = SourceLocation(file_name.str(), row, col, is_valid); + } else { + res = SourceLocation("", 0, 0, is_valid); + } + return res; +} + +} // namespace detail + } // namespace ast_canopy From 48d43f1ff14c044bc8af2065a0dafb43987ed4fe Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Wed, 6 Aug 2025 13:45:22 -0700 Subject: [PATCH 5/7] add typedef source loc --- ast_canopy/cpp/src/typedef.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ast_canopy/cpp/src/typedef.cpp b/ast_canopy/cpp/src/typedef.cpp index 5d0ad04e..899886c3 100644 --- a/ast_canopy/cpp/src/typedef.cpp +++ b/ast_canopy/cpp/src/typedef.cpp @@ -14,7 +14,8 @@ namespace ast_canopy { Typedef::Typedef(const clang::TypedefDecl *TD, - std::unordered_map *record_id_to_name) { + std::unordered_map *record_id_to_name) + : Decl(TD) { name = TD->getNameAsString(); clang::QualType qd = TD->getUnderlyingType(); clang::RecordDecl *underlying_record_decl = qd->getAsCXXRecordDecl(); From f4171b5b5ce9bc8c20979cd2e6be2f5eb254aa08 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Wed, 6 Aug 2025 13:46:50 -0700 Subject: [PATCH 6/7] amend test expects --- ast_canopy/tests/data/sample_source_loc.cu | 6 +++--- ast_canopy/tests/test_sample_source_loc.py | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/ast_canopy/tests/data/sample_source_loc.cu b/ast_canopy/tests/data/sample_source_loc.cu index eeb9dbf8..98a5c308 100644 --- a/ast_canopy/tests/data/sample_source_loc.cu +++ b/ast_canopy/tests/data/sample_source_loc.cu @@ -5,8 +5,8 @@ void __device__ __forceinline__ foo() {} // line 6 -struct Bar { // line 8 - Bar() {} // line 9 +struct Watermelon { // line 8 + Watermelon() {} // line 9 }; template // line 12 @@ -15,7 +15,7 @@ void __device__ __forceinline__ baz() {} // line 13 template // line 15 struct Bax {}; // line 16 -enum class Watermelon {}; // line 18 +enum class Fruit {}; // line 18 typedef Watermelon Suika; // line 20 diff --git a/ast_canopy/tests/test_sample_source_loc.py b/ast_canopy/tests/test_sample_source_loc.py index 8931fb22..ba5c794d 100644 --- a/ast_canopy/tests/test_sample_source_loc.py +++ b/ast_canopy/tests/test_sample_source_loc.py @@ -34,10 +34,11 @@ def test_source_location(data_folder): assert barctor.source_location.column == 5 assert barctor.source_location.is_valid + # The template has the same source location as the function baz = decls.function_templates[0] assert "sample_source_loc.cu" in baz.source_location.file_name - assert baz.source_location.line == 12 - assert baz.source_location.column == 0 + assert baz.source_location.line == 13 + assert baz.source_location.column == 33 assert baz.source_location.is_valid assert "sample_source_loc.cu" in baz.function.source_location.file_name @@ -45,10 +46,11 @@ def test_source_location(data_folder): assert baz.function.source_location.column == 33 assert baz.function.source_location.is_valid + # The template has the same source location as the class bax = decls.class_templates[0] assert "sample_source_loc.cu" in bax.source_location.file_name - assert bax.source_location.line == 15 - assert bax.source_location.column == 0 + assert bax.source_location.line == 16 + assert bax.source_location.column == 8 assert bax.source_location.is_valid assert "sample_source_loc.cu" in bax.record.source_location.file_name @@ -56,14 +58,14 @@ def test_source_location(data_folder): assert bax.record.source_location.column == 8 assert bax.record.source_location.is_valid - watermelon = decls.enum_classes[0] + watermelon = decls.enums[0] assert "sample_source_loc.cu" in watermelon.source_location.file_name assert watermelon.source_location.line == 18 - assert watermelon.source_location.column == 0 + assert watermelon.source_location.column == 12 assert watermelon.source_location.is_valid suika = decls.typedefs[0] assert "sample_source_loc.cu" in suika.source_location.file_name - assert suika.source_location.line == 19 - assert suika.source_location.column == 0 + assert suika.source_location.line == 20 + assert suika.source_location.column == 20 assert suika.source_location.is_valid From 1a556d55f3901ad896a3950bb10636af9c1fd23f Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Wed, 6 Aug 2025 14:04:33 -0700 Subject: [PATCH 7/7] regen pylibastcanopy stub --- ast_canopy/ast_canopy/pylibastcanopy.pyi | 45 ++++++++++++++++++------ 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/ast_canopy/ast_canopy/pylibastcanopy.pyi b/ast_canopy/ast_canopy/pylibastcanopy.pyi index 8cacf45a..417c92d6 100644 --- a/ast_canopy/ast_canopy/pylibastcanopy.pyi +++ b/ast_canopy/ast_canopy/pylibastcanopy.pyi @@ -1,17 +1,24 @@ from _typeshed import Incomplete from typing import ClassVar, overload -class ClassTemplate(Template): +class ClassTemplate(Decl, Template): num_min_required_args: int record: Incomplete def __init__(self, *args, **kwargs) -> None: ... -class ConstExprVar: +class ConstExprVar(Decl): name: str type_: Type value: str def __init__(self) -> None: ... +class Decl: + source_location: SourceLocation + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, arg0) -> None: ... + class Declarations: class_templates: list[ClassTemplate] enums: list[Enum] @@ -21,26 +28,28 @@ class Declarations: typedefs: list[Typedef] def __init__(self, *args, **kwargs) -> None: ... -class Enum: +class Enum(Decl): enumerator_values: list[str] enumerators: list[str] name: str def __init__(self, arg0) -> None: ... -class Field: +class Field(Decl): access: access_kind name: str type_: Type def __init__(self, *args, **kwargs) -> None: ... -class Function: +class Function(Decl): exec_space: execution_space + is_constexpr: bool + mangled_name: str name: str params: list[ParamVar] return_type: Type def __init__(self, *args, **kwargs) -> None: ... -class FunctionTemplate(Template): +class FunctionTemplate(Decl, Template): function: Function num_min_required_args: int def __init__(self, *args, **kwargs) -> None: ... @@ -50,12 +59,14 @@ class Method(Function): def __init__(self, *args, **kwargs) -> None: ... def is_move_constructor(self) -> bool: ... -class ParamVar: +class ParamVar(Decl): name: str type_: Type def __init__(self, arg0: str, arg1: Type) -> None: ... -class Record: +class ParseError(Exception): ... + +class Record(Decl): alignof_: int fields: list[Field] methods: list[Method] @@ -66,12 +77,26 @@ class Record: templated_methods: list[FunctionTemplate] def __init__(self, *args, **kwargs) -> None: ... +class SourceLocation: + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, arg0: str, arg1: int, arg2: int, arg3: bool) -> None: ... + @property + def column(self) -> int: ... + @property + def file_name(self) -> str: ... + @property + def is_valid(self) -> bool: ... + @property + def line(self) -> int: ... + class Template: num_min_required_args: int template_parameters: list[TemplateParam] def __init__(self, arg0: list[TemplateParam], arg1: int) -> None: ... -class TemplateParam: +class TemplateParam(Decl): kind: template_param_kind name: str type_: Type @@ -89,7 +114,7 @@ class Type: def is_left_reference(self) -> bool: ... def is_right_reference(self) -> bool: ... -class Typedef: +class Typedef(Decl): name: str underlying_name: str def __init__(self, *args, **kwargs) -> None: ...