PhotonPoseEstimator.h

Go to the documentation of this file.

/*
 * MIT License
 *
 * Copyright (c) PhotonVision
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
 
#pragma once
 
#include <optional>
#include <span>
 
#include <wpi/apriltag/AprilTagFieldLayout.hpp>
#include <wpi/math/geometry/Pose3d.hpp>
#include <wpi/math/geometry/Rotation3d.hpp>
#include <wpi/math/geometry/Transform3d.hpp>
#include <wpi/math/interpolation/TimeInterpolatableBuffer.hpp>
#include <wpi/util/SmallVector.hpp>
 
#include "photon/PhotonCamera.h"
#include "photon/targeting/PhotonPipelineResult.h"
#include "photon/targeting/PhotonTrackedTarget.h"
 
namespace photon {
enum PoseStrategy {
  LOWEST_AMBIGUITY = 0,
  CLOSEST_TO_CAMERA_HEIGHT,
  CLOSEST_TO_REFERENCE_POSE,
  CLOSEST_TO_LAST_POSE,
  AVERAGE_BEST_TARGETS,
  MULTI_TAG_PNP_ON_COPROCESSOR,
  MULTI_TAG_PNP_ON_RIO,
  CONSTRAINED_SOLVEPNP,
  PNP_DISTANCE_TRIG_SOLVE
};
 
struct ConstrainedSolvepnpParams {
  bool headingFree{false};
  double headingScalingFactor{0.0};
};
 
struct EstimatedRobotPose {
  /** The estimated pose */
  wpi::math::Pose3d estimatedPose;
  /** The estimated time the frame used to derive the robot pose was taken, in
   * the same timebase as the RoboRIO FPGA Timestamp */
  wpi::units::second_t timestamp;
 
  /** A list of the targets used to compute this pose */
  wpi::util::SmallVector<PhotonTrackedTarget, 10> targetsUsed;
 
  /** The strategy actually used to produce this pose */
  PoseStrategy strategy;
 
  EstimatedRobotPose(wpi::math::Pose3d pose_, wpi::units::second_t time_,
                     std::span<const PhotonTrackedTarget> targets,
                     PoseStrategy strategy_)
      : estimatedPose(pose_),
        timestamp(time_),
        targetsUsed(targets.data(), targets.data() + targets.size()),
        strategy(strategy_) {}
};
 
/**
 * The PhotonPoseEstimator class filters or combines readings from all the
 * fiducials visible at a given timestamp on the field to produce a single robot
 * in field pose, using the strategy set below. Example usage can be found in
 * our apriltagExample example project.
 */
class PhotonPoseEstimator {
 public:
  /**
   * Create a new PhotonPoseEstimator.
   *
   * @param aprilTags A AprilTagFieldLayout linking AprilTag IDs to Pose3ds with
   * respect to the FIRST field.
   * @param robotToCamera Transform3d from the center of the robot to the camera
   * mount positions (ie, robot ➔ camera).
   */
  explicit PhotonPoseEstimator(wpi::apriltag::AprilTagFieldLayout aprilTags,
                               wpi::math::Transform3d robotToCamera);
 
  /**
   * Get the AprilTagFieldLayout being used by the PositionEstimator.
   *
   * @return the AprilTagFieldLayout
   */
  wpi::apriltag::AprilTagFieldLayout GetFieldLayout() const {
    return aprilTags;
  }
 
  /**
   * @return The current transform from the center of the robot to the camera
   *         mount position.
   */
  inline wpi::math::Transform3d GetRobotToCameraTransform() {
    return m_robotToCamera;
  }
 
  /**
   * Useful for pan and tilt mechanisms, or cameras on turrets
   *
   * @param robotToCamera The current transform from the center of the robot to
   * the camera mount position.
   */
  inline void SetRobotToCameraTransform(wpi::math::Transform3d robotToCamera) {
    m_robotToCamera = robotToCamera;
  }
 
  /**
   * Add robot heading data to the buffer. Must be called periodically for the
   * PNP_DISTANCE_TRIG_SOLVE strategy.
   *
   * @param timestamp Timestamp of the robot heading data.
   * @param heading Field-relative heading at the given timestamp. Standard
   * WPILIB field coordinates.
   */
  inline void AddHeadingData(wpi::units::second_t timestamp,
                             wpi::math::Rotation2d heading) {
    this->headingBuffer.AddSample(timestamp, heading);
  }
 
  /**
   * Add robot heading data to the buffer. Must be called periodically for the
   * PNP_DISTANCE_TRIG_SOLVE strategy.
   *
   * @param timestamp Timestamp of the robot heading data.
   * @param heading Field-relative heading at the given timestamp. Standard
   * WPILIB coordinates.
   */
  inline void AddHeadingData(wpi::units::second_t timestamp,
                             wpi::math::Rotation3d heading) {
    AddHeadingData(timestamp, heading.ToRotation2d());
  }
 
  /**
   * Clears all heading data in the buffer, and adds a new seed. Useful for
   * preventing estimates from utilizing heading data provided prior to a pose
   * or rotation reset.
   *
   * @param timestamp Timestamp of the robot heading data.
   * @param heading Field-relative robot heading at given timestamp. Standard
   * WPILIB field coordinates.
   */
  inline void ResetHeadingData(wpi::units::second_t timestamp,
                               wpi::math::Rotation2d heading) {
    headingBuffer.Clear();
    AddHeadingData(timestamp, heading);
  }
 
  /**
   * Clears all heading data in the buffer, and adds a new seed. Useful for
   * preventing estimates from utilizing heading data provided prior to a pose
   * or rotation reset.
   *
   * @param timestamp Timestamp of the robot heading data.
   * @param heading Field-relative robot heading at given timestamp. Standard
   * WPILIB field coordinates.
   */
  inline void ResetHeadingData(wpi::units::second_t timestamp,
                               wpi::math::Rotation3d heading) {
    ResetHeadingData(timestamp, heading.ToRotation2d());
  }
 
  /**
   * Return the estimated position of the robot with the lowest position
   * ambiguity from a List of pipeline results.
   *
   * @param cameraResult A pipeline result from the camera.
   * @return An EstimatedRobotPose with an estimated pose, timestamp, and
   * targets used to create the estimate, or std::nullopt if there's no targets.
   */
  std::optional<EstimatedRobotPose> EstimateLowestAmbiguityPose(
      PhotonPipelineResult cameraResult);
 
  /**
   * Return the estimated position of the robot using the target with the lowest
   * delta height difference between the estimated and actual height of the
   * camera.
   *
   * @param cameraResult A pipeline result from the camera.
   * @return An EstimatedRobotPose with an estimated pose, timestamp and
   * targets used to create the estimate, or std::nullopt if there's no targets.
   */
  std::optional<EstimatedRobotPose> EstimateClosestToCameraHeightPose(
      PhotonPipelineResult cameraResult);
 
  /**
   * Return the estimated position of the robot using the target with the lowest
   * delta in the vector magnitude between it and the reference pose.
   *
   * @param cameraResult A pipeline result from the camera.
   * @param referencePose reference pose to check vector magnitude difference
   * against.
   * @return An EstimatedRobotPose with an estimated pose, timestamp, and
   * targets used to create the estimate, or std::nullopt if there's no targets.
   */
  std::optional<EstimatedRobotPose> EstimateClosestToReferencePose(
      PhotonPipelineResult cameraResult, wpi::math::Pose3d referencePose);
 
  /**
   * Return the estimated position of the robot by using all visible tags to
   * compute a single pose estimate on coprocessor. This option needs to be
   * enabled on the PhotonVision web UI as well.
   *
   * @param cameraResult A pipeline result from the camera.
   * @return An EstimatedRobotPose with an estimated pose, timestamp, and
   * targets used to create the estimate or std::nullopt if there's no targets,
   * no multi-tag results, or multi-tag is disabled in the web UI.
   */
  std::optional<EstimatedRobotPose> EstimateCoprocMultiTagPose(
      PhotonPipelineResult cameraResult);
 
  /**
   * Return the estimated position of the robot by using all visible tags to
   * compute a single pose estimate on the RoboRIO. This can take a lot of time
   * due to the RIO's weak computing power.
   *
   * @param cameraResult A pipeline result from the camera.
   * @param cameraMatrix Camera intrinsics from camera calibration data.
   * @param distCoeffs Distortion coefficients from camera calibration data.
   * @return An EstimatedRobotPose with an estimated pose, timestamp, and
   * targets used to create the estimate, or std::nullopt if there's less than 2
   * targets visible or SolvePnP fails.
   */
  std::optional<EstimatedRobotPose> EstimateRioMultiTagPose(
      PhotonPipelineResult cameraResult, PhotonCamera::CameraMatrix camMat,
      PhotonCamera::DistortionMatrix distCoeffs);
 
  /**
   * Return the estimated position of the robot by using distance data from best
   * visible tag to compute a Pose. This runs on the RoboRIO in order to access
   * the robot's yaw heading, and MUST have AddHeadingData called every frame so
   * heading data is up-to-date.
   *
   * <p>Yields a Pose2d in estimatedRobotPose (0 for z, roll, pitch)
   *
   * <p>https://www.chiefdelphi.com/t/frc-6328-mechanical-advantage-2025-build-thread/477314/98
   *
   * @param cameraResult A pipeline result from the camera.
   * @return An EstimatedRobotPose with an estimated pose, timestamp, and
   * targets used to create the estimate, or std::nullopt if there's no targets
   * or heading data.
   */
  std::optional<EstimatedRobotPose> EstimatePnpDistanceTrigSolvePose(
      PhotonPipelineResult cameraResult);
 
  /**
   * Return the average of the best target poses using ambiguity as weight.
   * @param cameraResult A pipeline result from the camera.
   * @return An EstimatedRobotPose with an estimated pose, timestamp, and
   * targets used to create the estimate, or std::nullopt if there's no targets.
   */
  std::optional<EstimatedRobotPose> EstimateAverageBestTargetsPose(
      PhotonPipelineResult cameraResult);
 
  /**
   * Return the estimated position of the robot by solving a constrained version
   * of the Perspective-n-Point problem with the robot's drivebase flat on the
   * floor. This computation takes place on the RoboRIO, and typically takes not
   * more than 2ms. See
   * photon::VisionEstimation::EstimateRobotPoseConstrainedSolvePNP for tuning
   * handles this strategy exposes. Internally, the cost function is a
   * sum-squared of pixel reprojection error + (optionally) heading error *
   * heading scale factor. This strategy needs addHeadingData called every frame
   * so heading data is up-to-date.
   *
   * @param cameraResult A pipeline result from the camera.
   * @param cameraMatrix Camera intrinsics from camera calibration data.
   * @param distCoeffs Distortion coefficients from camera calibration data.
   * @param seedPose An initial guess at robot pose, refined via optimization.
   * Better guesses will converge faster. Can come from any pose source, but
   * some battle-tested sources are estimateCoprocMultiTagPose, or
   * estimateLowestAmbiguityPose if MultiTag results aren't available.
   * @param headingFree If true, heading is completely free to vary. If false,
   * heading excursions from the provided heading measurement will be penalized
   * @param headingScaleFactor If headingFree is false, this weights the cost of
   * changing our robot heading estimate against the tag corner reprojection
   * error cost.
   * @return An EstimatedRobotPose with an estimated pose, timestamp, and
   * targets used to create the estimate, or std::nullopt if there's no targets
   * or heading data, or if the solver fails to solve the problem.
   */
  std::optional<EstimatedRobotPose> EstimateConstrainedSolvepnpPose(
      photon::PhotonPipelineResult cameraResult,
      photon::PhotonCamera::CameraMatrix cameraMatrix,
      photon::PhotonCamera::DistortionMatrix distCoeffs,
      wpi::math::Pose3d seedPose, bool headingFree, double headingScaleFactor);
 
 private:
  wpi::apriltag::AprilTagFieldLayout aprilTags;
 
  wpi::math::Transform3d m_robotToCamera;
 
  wpi::math::TimeInterpolatableBuffer<wpi::math::Rotation2d> headingBuffer;
 
  inline static int InstanceCount = 1;
};
 
}  // namespace photon