Hello everybody, right now I'm making a 3d Platformer in unity where the player has to roll a sphere around sort of like Katamari Damacy. I've already implemented the features needed to have my player roll the sphere, but I am sort of hitting a problem where whenever I move or jump with the sphere it keeps fidgeting in place(see video). I tried lowering/increasing the speed and the player and sphere to have a stable connection, but nothing to avail, any ideas on how I can solve this problem? Also here's the code on how I implemented it:

Progress: https://streamable.com/75ypd0

Video of what I am trying to obtain:

https://www.youtube.com/watch?v=AzWG10U7OSc(edited)

The Sphere's Physics Script:

public class PhysicsObject : MonoBehaviour
{
   public float waitOnPickup = 0.2f;
   public float breakForce = 35f;
   [HideInInspector] public bool pickedUp = false;
   [HideInInspector] public AttachBall2 playerInteractions;
   [SerializeField] public int gumTotal = 25;
   public ActivatePowerUp gumCounter;

   private void Start()
   {
       gumCounter = FindObjectOfType<ActivatePowerUp>();
   }
   private void OnCollisionEnter(Collision collision)
   {
       if (pickedUp)
       {
           if (collision.relativeVelocity.magnitude > breakForce)
           {
               playerInteractions.BreakConnection();
           }

           if(collision.gameObject.tag == "GumSplat")
           {
               Destroy(collision.gameObject);
               gumCounter.gumSplatsCollected += 1;
           }

       }
   }

   //this is used to prevent the connection from breaking when you just picked up the object as it sometimes fires a collision with the ground or whatever it is touching
   public IEnumerator PickUp()
   {
       yield return new WaitForSecondsRealtime(waitOnPickup);
       pickedUp = true;

   }
}

For trying to hold the sphere:

public class AttachBall2 : MonoBehaviour
{
   [Header("InteractableInfo")]
   public float sphereCastRadius = 0.5f;
   public int interactableLayerIndex;
   private Vector3 raycastPos;
   public GameObject lookObject;
   private PhysicsObject physicsObject;

   [Header("Pickup")]
   [SerializeField] private Transform pickupParent;
   public GameObject currentlyPickedUpObject;
   private Rigidbody pickupRB;
   ActivatePowerUp toActivate;

   [Header("ObjectFollow")]
   [SerializeField] private float minSpeed = 0;
   [SerializeField] private float maxSpeed = 300f;
   [SerializeField] private float maxDistance = 10f;
   private float currentSpeed = 0f;
   private float currentDist = 0f;
   [SerializeField] public float followSpeed = 20f;
   PlayerBhysics playerSpeed;

   [Header("Rotation")]
   public float rotationSpeed = 100f;
   Quaternion lookRot;

   [Header("Player Speed")]
   public float playerTopSpeed;
   public float playerAccel;
   public float playerSlopeLimit;


   private void Start()
   {
       toActivate = GetComponent<ActivatePowerUp>();
   }
   //A simple visualization of the point we're following in the scene view
   private void OnDrawGizmos()
   {
       Gizmos.color = Color.yellow;
       Gizmos.DrawSphere(pickupParent.position, 0.5f);
   }

   //Interactable Object detections and distance check
   void Update()
   {
       //Here we check if we're currently looking at an interactable object
       RaycastHit hit;
       if (Physics.SphereCast(transform.position, sphereCastRadius, transform.forward, out hit, maxDistance, 1 << interactableLayerIndex))
       {

           lookObject = hit.collider.transform.root.gameObject;
           //Debug.Log(lookObject);

       }
       else
       {
           lookObject = null;

       }



       //if we press the button of choice
       if (Input.GetButtonDown("Attach"))
       {
           //and we're not holding anything
           if (currentlyPickedUpObject == null)
           {
               //and we are looking an interactable object
               if (lookObject != null)
               {
                   Debug.Log("Picked up");
                   PickUpObject();
                   toActivate.hasGumball = true;
               }

           }
           //if we press the pickup button and have something, we drop it
           else
           {
               BreakConnection();
               toActivate.hasGumball = false;
           }
       }


   }

   ////Velocity movement toward pickup parent and rotation
   private void FixedUpdate()
   {
       if (currentlyPickedUpObject != null)
       {
           currentDist = Vector3.Distance(pickupParent.position, pickupRB.position);
           //Calculate the current speed
           currentSpeed = Mathf.SmoothStep(minSpeed, maxSpeed, currentDist / maxDistance);
           currentSpeed *= Time.fixedDeltaTime;
           Vector3 direction = pickupParent.position - pickupRB.position;
           pickupRB.velocity = direction.normalized * (followSpeed * 100f * Time.deltaTime);
           //Rotation
           lookRot = Quaternion.LookRotation(transform.position - pickupRB.position);
           lookRot = Quaternion.Slerp(transform.rotation, lookRot, rotationSpeed * Time.fixedDeltaTime);
           pickupRB.MoveRotation(lookRot);
       }

   }

   ////Release the object
   public void BreakConnection()
   {
       pickupRB.constraints = RigidbodyConstraints.None;
       currentlyPickedUpObject = null;
       physicsObject.pickedUp = false;
       currentDist = 0;
   }

   public void PickUpObject()
   {
       physicsObject = lookObject.GetComponentInChildren<PhysicsObject>();
       currentlyPickedUpObject = lookObject;
       pickupRB = currentlyPickedUpObject.GetComponent<Rigidbody>();
       pickupRB.constraints = RigidbodyConstraints.FreezeRotation;
       physicsObject.playerInteractions = this;
       StartCoroutine(physicsObject.PickUp());

       //Increase Player Speed
       movSpeed.TopSpeed = playerTopSpeed;
       movSpeed.MoveAccell = playerAccel;
       movSpeed.SlopeSpeedLimit = playerSlopeLimit;


   }